├── .github └── workflows │ └── daily-rebuild.yml ├── .gitignore ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── dats-site.py ├── img ├── clrmamepro.png └── headers.png ├── no-intro.py ├── redump.py └── smdb.py /.github/workflows/daily-rebuild.yml: -------------------------------------------------------------------------------- 1 | name: Daily rebuild 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | schedule: 8 | - cron: '0 12 * * *' 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | - uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.12' 20 | cache: 'pipenv' 21 | 22 | - name: Setup base environment 23 | run: | 24 | echo "timestamp=$(date +%s)" >> $GITHUB_ENV 25 | 26 | - name: Install pipenv 27 | run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python 28 | - name: Setup pipenv environment 29 | run: pipenv install --deploy 30 | 31 | - name: Download dats and generate profile.xml for Redump 32 | if: ${{ ! cancelled() }} 33 | run: pipenv run python3 redump.py 34 | 35 | - name: Download dats and generate profile.xml for No-Intro 36 | if: ${{ ! cancelled() }} 37 | run: pipenv run python3 no-intro.py 38 | 39 | # HTGD disabled for now 40 | #- name: Download dats and generate profile.xml for HTGD 41 | # if: ${{ ! cancelled() }} 42 | # run: pipenv run python3 smdb.py 43 | 44 | # dats.site disabled for now 45 | #- name: Download dats and generate profile.xml for dats.site 46 | # if: ${{ ! cancelled() }} 47 | # run: pipenv run python3 dats-site.py 48 | 49 | - name: Create release 50 | if: ${{ ! cancelled() }} 51 | uses: ncipollo/release-action@v1 52 | with: 53 | allowUpdates: true 54 | artifacts: "*.xml,*.zip" 55 | commit: master 56 | name: Daily_Rebuild 57 | makeLatest: true 58 | tag: Daily_Rebuild 59 | token: ${{ secrets.GITHUB_TOKEN }} 60 | body: "Automatically released by github actions" 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.xml 3 | *.dat 4 | *.zip 5 | .DS_Store 6 | EverDrive-Packs-Lists-Database/ 7 | geckodriver.log 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | # redump.py, dats-site.py, smdb.py 10 | requests = "*" 11 | # no-intro.py 12 | selenium = "*" 13 | # smdb.py 14 | lxml = "*" 15 | 16 | [requires] 17 | python_version = "3.12" 18 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "0044f8a6e4570e9a988ef631541d98cbd3ca7e3254090fdedd639fdac30178df" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.12" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "attrs": { 20 | "hashes": [ 21 | "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", 22 | "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308" 23 | ], 24 | "markers": "python_version >= '3.8'", 25 | "version": "==24.3.0" 26 | }, 27 | "certifi": { 28 | "hashes": [ 29 | "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", 30 | "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" 31 | ], 32 | "markers": "python_version >= '3.6'", 33 | "version": "==2024.12.14" 34 | }, 35 | "charset-normalizer": { 36 | "hashes": [ 37 | "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", 38 | "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", 39 | "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", 40 | "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", 41 | "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", 42 | "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", 43 | "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", 44 | "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", 45 | "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", 46 | "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", 47 | "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", 48 | "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", 49 | "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", 50 | "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", 51 | "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", 52 | "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", 53 | "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", 54 | "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", 55 | "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", 56 | "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", 57 | "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", 58 | "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", 59 | "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", 60 | "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", 61 | "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", 62 | "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", 63 | "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", 64 | "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", 65 | "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", 66 | "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", 67 | "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", 68 | "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", 69 | "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", 70 | "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", 71 | "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", 72 | "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", 73 | "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", 74 | "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", 75 | "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", 76 | "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", 77 | "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", 78 | "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", 79 | "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", 80 | "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", 81 | "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", 82 | "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", 83 | "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", 84 | "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", 85 | "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", 86 | "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", 87 | "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", 88 | "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", 89 | "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", 90 | "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", 91 | "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", 92 | "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", 93 | "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", 94 | "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", 95 | "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", 96 | "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", 97 | "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", 98 | "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", 99 | "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", 100 | "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", 101 | "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", 102 | "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", 103 | "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", 104 | "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", 105 | "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", 106 | "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", 107 | "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", 108 | "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", 109 | "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", 110 | "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", 111 | "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", 112 | "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", 113 | "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", 114 | "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", 115 | "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", 116 | "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", 117 | "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", 118 | "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", 119 | "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", 120 | "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", 121 | "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", 122 | "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", 123 | "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", 124 | "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", 125 | "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", 126 | "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", 127 | "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", 128 | "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", 129 | "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", 130 | "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", 131 | "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", 132 | "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", 133 | "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", 134 | "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", 135 | "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", 136 | "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", 137 | "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", 138 | "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", 139 | "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", 140 | "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", 141 | "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" 142 | ], 143 | "markers": "python_full_version >= '3.7.0'", 144 | "version": "==3.4.0" 145 | }, 146 | "h11": { 147 | "hashes": [ 148 | "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", 149 | "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" 150 | ], 151 | "index": "pypi", 152 | "markers": "python_version >= '3.8'", 153 | "version": "==0.16.0" 154 | }, 155 | "idna": { 156 | "hashes": [ 157 | "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", 158 | "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" 159 | ], 160 | "markers": "python_version >= '3.6'", 161 | "version": "==3.10" 162 | }, 163 | "lxml": { 164 | "hashes": [ 165 | "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e", 166 | "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229", 167 | "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3", 168 | "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5", 169 | "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70", 170 | "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15", 171 | "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002", 172 | "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd", 173 | "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22", 174 | "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf", 175 | "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22", 176 | "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832", 177 | "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727", 178 | "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e", 179 | "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30", 180 | "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f", 181 | "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f", 182 | "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51", 183 | "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4", 184 | "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de", 185 | "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875", 186 | "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42", 187 | "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e", 188 | "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6", 189 | "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391", 190 | "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc", 191 | "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b", 192 | "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237", 193 | "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4", 194 | "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86", 195 | "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f", 196 | "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a", 197 | "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8", 198 | "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f", 199 | "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903", 200 | "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03", 201 | "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e", 202 | "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99", 203 | "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7", 204 | "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab", 205 | "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d", 206 | "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22", 207 | "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492", 208 | "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b", 209 | "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3", 210 | "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be", 211 | "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469", 212 | "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f", 213 | "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a", 214 | "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c", 215 | "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a", 216 | "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4", 217 | "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94", 218 | "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442", 219 | "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b", 220 | "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84", 221 | "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c", 222 | "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9", 223 | "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1", 224 | "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be", 225 | "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367", 226 | "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e", 227 | "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21", 228 | "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa", 229 | "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16", 230 | "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d", 231 | "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe", 232 | "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83", 233 | "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", 234 | "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040", 235 | "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763", 236 | "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8", 237 | "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff", 238 | "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2", 239 | "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a", 240 | "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b", 241 | "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce", 242 | "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c", 243 | "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577", 244 | "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8", 245 | "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71", 246 | "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512", 247 | "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540", 248 | "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f", 249 | "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2", 250 | "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a", 251 | "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce", 252 | "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e", 253 | "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2", 254 | "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27", 255 | "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1", 256 | "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d", 257 | "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1", 258 | "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330", 259 | "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920", 260 | "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99", 261 | "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff", 262 | "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18", 263 | "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff", 264 | "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c", 265 | "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179", 266 | "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080", 267 | "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19", 268 | "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d", 269 | "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70", 270 | "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32", 271 | "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a", 272 | "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2", 273 | "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79", 274 | "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3", 275 | "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5", 276 | "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f", 277 | "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d", 278 | "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3", 279 | "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b", 280 | "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753", 281 | "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9", 282 | "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957", 283 | "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033", 284 | "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb", 285 | "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656", 286 | "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab", 287 | "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b", 288 | "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d", 289 | "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd", 290 | "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859", 291 | "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11", 292 | "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c", 293 | "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a", 294 | "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005", 295 | "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654", 296 | "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80", 297 | "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e", 298 | "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec", 299 | "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7", 300 | "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965", 301 | "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945", 302 | "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8" 303 | ], 304 | "index": "pypi", 305 | "markers": "python_version >= '3.6'", 306 | "version": "==5.3.0" 307 | }, 308 | "outcome": { 309 | "hashes": [ 310 | "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", 311 | "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b" 312 | ], 313 | "markers": "python_version >= '3.7'", 314 | "version": "==1.3.0.post0" 315 | }, 316 | "pysocks": { 317 | "hashes": [ 318 | "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299", 319 | "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", 320 | "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" 321 | ], 322 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 323 | "version": "==1.7.1" 324 | }, 325 | "requests": { 326 | "hashes": [ 327 | "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", 328 | "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" 329 | ], 330 | "index": "pypi", 331 | "markers": "python_version >= '3.8'", 332 | "version": "==2.32.3" 333 | }, 334 | "selenium": { 335 | "hashes": [ 336 | "sha256:5296c425a75ff1b44d0d5199042b36a6d1ef76c04fb775b97b40be739a9caae2", 337 | "sha256:b89b1f62b5cfe8025868556fe82360d6b649d464f75d2655cb966c8f8447ea18" 338 | ], 339 | "index": "pypi", 340 | "markers": "python_version >= '3.8'", 341 | "version": "==4.27.1" 342 | }, 343 | "sniffio": { 344 | "hashes": [ 345 | "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", 346 | "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" 347 | ], 348 | "markers": "python_version >= '3.7'", 349 | "version": "==1.3.1" 350 | }, 351 | "sortedcontainers": { 352 | "hashes": [ 353 | "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", 354 | "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" 355 | ], 356 | "version": "==2.4.0" 357 | }, 358 | "trio": { 359 | "hashes": [ 360 | "sha256:1dcc95ab1726b2da054afea8fd761af74bad79bd52381b84eae408e983c76831", 361 | "sha256:68eabbcf8f457d925df62da780eff15ff5dc68fd6b367e2dde59f7aaf2a0b884" 362 | ], 363 | "markers": "python_version >= '3.8'", 364 | "version": "==0.27.0" 365 | }, 366 | "trio-websocket": { 367 | "hashes": [ 368 | "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f", 369 | "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638" 370 | ], 371 | "markers": "python_version >= '3.7'", 372 | "version": "==0.11.1" 373 | }, 374 | "typing-extensions": { 375 | "hashes": [ 376 | "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", 377 | "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" 378 | ], 379 | "markers": "python_version >= '3.8'", 380 | "version": "==4.12.2" 381 | }, 382 | "urllib3": { 383 | "extras": [ 384 | "socks" 385 | ], 386 | "hashes": [ 387 | "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", 388 | "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" 389 | ], 390 | "markers": "python_version >= '3.8'", 391 | "version": "==2.2.3" 392 | }, 393 | "websocket-client": { 394 | "hashes": [ 395 | "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", 396 | "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da" 397 | ], 398 | "markers": "python_version >= '3.8'", 399 | "version": "==1.8.0" 400 | }, 401 | "wsproto": { 402 | "hashes": [ 403 | "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", 404 | "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" 405 | ], 406 | "markers": "python_full_version >= '3.7.0'", 407 | "version": "==1.2.0" 408 | } 409 | }, 410 | "develop": {} 411 | } 412 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Auto DAT file generator 2 | 3 | ![Daily Rebuild Status](https://github.com/hugo19941994/auto-datfile-generator/actions/workflows/daily-rebuild.yml/badge.svg) 4 | 5 | WWW profiles to use in clrmamepro for the standard No-Intro and Redump sets. 6 | Refreshes once every 24h automatically. 7 | 8 | ## URLs 9 | 10 | ### No-Intro 11 | 12 | `https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/no-intro.xml` 13 | 14 | ### No-Intro (parent-clone) 15 | 16 | `https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/no-intro_parent-clone.xml` 17 | 18 | ### Redump 19 | 20 | `https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/redump.xml` 21 | 22 | 23 | 26 | 27 | 28 | 31 | 32 | ![clrmamepro screenshot](./img/clrmamepro.png) 33 | 34 | Project inspired by [redump-xml-updater](https://github.com/bilakispa/redump-xml-updater) 35 | 36 | ## Header support 37 | 38 | Some No-Intro dats require an extra XML file to detect headers. 39 | 40 | ![clrmamepro header warning screenshot](./img/headers.png) 41 | 42 | Download the following zips, extract them and place the XML files in clrmamepro's `headers` folder: 43 | 44 | - [Atari Jaguar](https://datomatic.no-intro.org/stuff/header_a7800.zip) 45 | - [Atari Lynx](https://datomatic.no-intro.org/stuff/header_lynx.zip) 46 | - [Nintendo FDS](https://datomatic.no-intro.org/stuff/header_fds.zip) 47 | - [Nintendo NES](https://datomatic.no-intro.org/stuff/header_nes.zip) 48 | -------------------------------------------------------------------------------- /dats-site.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from io import BytesIO, StringIO 3 | import os 4 | import re 5 | import requests 6 | import xml.etree.ElementTree as ET 7 | import zipfile 8 | 9 | xml_filename = 'dats-site.xml' 10 | 11 | regex = { 12 | 'date': r'\)_\((.*?)\)\.', 13 | 'name': r'filename=(.*?).zip', 14 | 'filename': r'filename=(.*)', 15 | } 16 | 17 | # zip file to store all DAT files 18 | zipObj = zipfile.ZipFile(f'dats-site.zip', 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=9) 19 | 20 | # clrmamepro XML file 21 | tag_clrmamepro = ET.Element('clrmamepro') 22 | 23 | 24 | for i in range(1, 8): 25 | print(f'Downloading dat {i}/7') 26 | response = requests.get(f"https://dats.site/getcustdat.php?custdatid={i}") 27 | content_header = response.headers['Content-Disposition'] 28 | print(content_header) 29 | 30 | # section for this dat in the XML file 31 | tag_datfile = ET.SubElement(tag_clrmamepro, 'datfile') 32 | 33 | # XML version 34 | dat_date = re.findall(regex['date'], content_header)[0] 35 | print(dat_date) 36 | ET.SubElement(tag_datfile, 'version').text = dat_date 37 | 38 | # XML name & description 39 | tempName = re.findall(regex['name'], content_header)[0] 40 | print(tempName) 41 | # trim the - from the end (if exists) 42 | if (tempName.endswith('-')): 43 | tempName = tempName[:-2] 44 | elif (tempName.endswith('BIOS')): 45 | tempName = tempName + ' Images' 46 | ET.SubElement(tag_datfile, 'name').text = tempName 47 | ET.SubElement(tag_datfile, 'description').text = tempName 48 | 49 | # URL tag in XML 50 | ET.SubElement( 51 | tag_datfile, 'url').text = f'https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/dats-site.zip' 52 | 53 | # File tag in XML 54 | originalFileName = re.findall(regex['filename'], content_header)[0] 55 | print(originalFileName) 56 | fileName = f'{originalFileName[:-4]}.dat' 57 | ET.SubElement(tag_datfile, 'file').text = fileName 58 | 59 | # Author tag in XML 60 | ET.SubElement(tag_datfile, 'author').text = 'dats.site' 61 | 62 | # Command XML tag 63 | ET.SubElement(tag_datfile, 'comment').text = '_' 64 | 65 | # Get the DAT file 66 | datfile_name = f'{fileName[:-4]}.dat' 67 | print(datfile_name) 68 | 69 | # extract datfile from zip to store in the DB zip 70 | zipdata = BytesIO() 71 | zipdata.write(response.content) 72 | archive = zipfile.ZipFile(zipdata) 73 | zipObj.writestr(datfile_name, archive.read(datfile_name)) 74 | 75 | # store clrmamepro XML file 76 | xmldata = ET.tostring(tag_clrmamepro).decode() 77 | xmlfile = open(xml_filename, 'w') 78 | xmlfile.write(xmldata) 79 | 80 | # Save DB zip file 81 | zipObj.close() 82 | -------------------------------------------------------------------------------- /img/clrmamepro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hugo19941994/auto-datfile-generator/49fa2a465871fa610f2bed7e613ac92016a94713/img/clrmamepro.png -------------------------------------------------------------------------------- /img/headers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hugo19941994/auto-datfile-generator/49fa2a465871fa610f2bed7e613ac92016a94713/img/headers.png -------------------------------------------------------------------------------- /no-intro.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import xml.etree.ElementTree as ET 4 | import zipfile 5 | from time import sleep 6 | 7 | from selenium import webdriver 8 | 9 | regex = { 10 | "date" : r"[0-9]{8}-[0-9]{6}", 11 | "name" : r"(.*?.)( \([0-9]{8}-[0-9]{6}\).dat)" 12 | } 13 | 14 | no_intro_type = { 15 | "standard" : "standard", 16 | "parent-clone" : "xml" 17 | } 18 | 19 | for key, value in no_intro_type.items(): 20 | 21 | # Download no-intro pack using selenium 22 | dir_path = os.path.dirname(os.path.realpath(__file__)) 23 | 24 | options = webdriver.FirefoxOptions() 25 | options.set_preference("browser.download.folderList", 2) 26 | options.set_preference("browser.download.manager.showWhenStarting", False) 27 | options.set_preference("browser.download.dir", dir_path) 28 | options.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/zip") 29 | options.add_argument ("-headless") 30 | 31 | service = webdriver.FirefoxService(log_output = "firefox-webdriver.log" , service_args = ["--log", "debug"]) 32 | 33 | driver = webdriver.Firefox(service=service, options=options) 34 | driver.implicitly_wait(10) 35 | 36 | # load website 37 | driver.get("https://datomatic.no-intro.org") 38 | print("Loaded no-intro datomatic ...") 39 | 40 | # select "DOWNLOAD" 41 | driver.find_element(by="xpath", value="/html/body/div/header/nav/ul/li[3]/a").click() 42 | 43 | # select "daily" 44 | driver.find_element(by="xpath", value="/html/body/div/section/article/table[1]/tbody/tr/td/a[5]").click() 45 | 46 | #set the type of dat file 47 | if key == "standard" : 48 | driver.find_element(by="xpath", value="//input[@name='dat_type' and @value='dat']").click() 49 | if key == "parent-clone" : 50 | driver.find_element(by="xpath", value="//input[@name='dat_type' and @value='pc']").click() 51 | print(f"Set dat type to {key} ...") 52 | 53 | # select "Request" 54 | driver.find_element(by="name", value="updated_daily").click() 55 | sleep(5) 56 | 57 | # select "Download" 58 | driver.find_element(by="xpath", value="//input[@value='Download!']").click() 59 | print("Waiting for download to complete ...") 60 | 61 | # wait until file is found 62 | FOUND = False 63 | NAME = None 64 | TIME_SLEPT = 0 65 | while not FOUND: 66 | if TIME_SLEPT > 900: 67 | raise FileNotFoundError(f"No-Intro {key} zip file not found, timeout reached") 68 | 69 | for f in os.listdir(dir_path): 70 | if "No-Intro Love Pack" in f and not f.endswith(".part"): 71 | try: 72 | zipfile.ZipFile(os.path.join(dir_path, f)) 73 | NAME = f 74 | FOUND = True 75 | print("No-Intro zip file download completed ...") 76 | break 77 | except zipfile.BadZipfile: 78 | pass 79 | 80 | # wait 5 seconds 81 | sleep(5) 82 | TIME_SLEPT += 5 83 | 84 | if NAME == None: 85 | raise FileNotFoundError(f"No-Intro {key} zip file not found, download failed") 86 | 87 | #setup archive path and rename 88 | archive_name = "no-intro.zip" if key == "standard" else f"no-intro_{key}.zip" 89 | archive_full = os.path.join(dir_path, archive_name) 90 | os.rename(os.path.join(dir_path, NAME), os.path.join(dir_path, archive_full)) 91 | 92 | # load & extract zip file, there is currently no way to remove files from zip archive 93 | with zipfile.ZipFile(os.path.join(dir_path, archive_full), mode="r") as orig_archive: 94 | orig_archive.extractall() 95 | # delete unneeded files 96 | os.remove("index.txt") 97 | 98 | print("Building new archive ...") 99 | with zipfile.ZipFile(os.path.join(dir_path, archive_full), mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: 100 | for f in os.listdir(dir_path): 101 | if "No-Intro" in f: 102 | print("\nAdding No-Intro dats ...") 103 | os.chdir("./No-Intro") 104 | for x in os.listdir(path="."): 105 | if x.endswith(".dat"): 106 | print("Adding to Archive: ", x) 107 | archive.write(x) 108 | os.remove(x) 109 | os.chdir("../") 110 | os.rmdir("./No-Intro") 111 | if "Non-Redump" in f: 112 | print("\nAdding No-Intro Non-Redump dats ...") 113 | os.chdir("./Non-Redump") 114 | for x in os.listdir(path="."): 115 | if x.endswith(".dat"): 116 | print("Adding to Archive: ", x) 117 | archive.write(x) 118 | os.remove(x) 119 | os.chdir("../") 120 | os.rmdir("./Non-Redump") 121 | if "Source Code" in f: 122 | print("\nAdding No-Intro Source Code dats ...") 123 | os.chdir("./Source Code") 124 | for x in os.listdir(path="."): 125 | if x.endswith(".dat"): 126 | print("Adding to Archive: ", x) 127 | archive.write(x) 128 | os.remove(x) 129 | os.chdir("../") 130 | os.rmdir("./Source Code") 131 | if "Unofficial" in f: 132 | print("\nAdding No-Intro Unofficial dats ...") 133 | os.chdir("./Unofficial") 134 | for x in os.listdir(path="."): 135 | if x.endswith(".dat"): 136 | print("Adding to Archive: ", x) 137 | archive.write(x) 138 | os.remove(x) 139 | os.chdir("../") 140 | os.rmdir("./Unofficial") 141 | 142 | print("\nCreating new clrmamepro datfile ...\n") 143 | # clrmamepro XML file 144 | tag_clrmamepro = ET.Element("clrmamepro") 145 | for dat in archive.namelist(): 146 | print(dat) 147 | # section for this dat in the XML file 148 | tag_datfile = ET.SubElement(tag_clrmamepro, "datfile") 149 | 150 | # XML version 151 | dat_date = re.findall(regex["date"], dat)[0] 152 | ET.SubElement(tag_datfile, "version").text = dat_date 153 | print(dat_date) 154 | 155 | # XML name & description 156 | tempName = re.findall(regex["name"], dat)[0][0] 157 | ET.SubElement(tag_datfile, "name").text = tempName 158 | ET.SubElement(tag_datfile, "description").text = tempName 159 | print(tempName) 160 | 161 | # URL tag in XML 162 | ET.SubElement(tag_datfile, "url").text = f"https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/{archive_name}" 163 | 164 | # File tag in XML 165 | fileName = dat 166 | fileName = f"{fileName[:-4]}.dat" 167 | ET.SubElement(tag_datfile, "file").text = fileName 168 | print(fileName) 169 | 170 | # Author tag in XML 171 | ET.SubElement(tag_datfile, "author").text = "no-intro.org" 172 | 173 | # Command XML tag 174 | ET.SubElement(tag_datfile, "comment").text = "_" 175 | 176 | print() 177 | 178 | archive.close() 179 | 180 | # store clrmamepro XML file 181 | xmldata = ET.tostring(tag_clrmamepro).decode() 182 | xml_filename = "no-intro.xml" if key == "standard" else f"no-intro_{key}.xml" 183 | 184 | with open(xml_filename, "w", encoding="utf-8") as xmlfile: 185 | xmlfile.write(xmldata) 186 | 187 | print("Finished") 188 | -------------------------------------------------------------------------------- /redump.py: -------------------------------------------------------------------------------- 1 | import re 2 | import xml.etree.ElementTree as ET 3 | import zipfile 4 | from io import BytesIO 5 | from time import sleep 6 | import requests 7 | 8 | # Config 9 | URL_HOME = "http://redump.org/" 10 | URL_DOWNLOADS = "http://redump.org/downloads/" 11 | XML_FILENAME = "redump.xml" 12 | XML_URL = "https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/redump.zip" 13 | 14 | regex = { 15 | "datfile" : r'', 16 | "date" : r"\) \((.*?)\)\.", 17 | "name" : r'filename="(.*?) Datfile', 18 | "filename" : r'filename="(.*?)"', 19 | } 20 | 21 | 22 | def _find_dats(): 23 | download_page = requests.get(URL_DOWNLOADS, timeout=150) 24 | download_page.raise_for_status() 25 | 26 | dat_files = re.findall(regex["datfile"], download_page.text) 27 | return dat_files 28 | 29 | 30 | def update_XML(): 31 | dat_list = _find_dats() 32 | 33 | # zip file to store all DAT files 34 | zip_object = zipfile.ZipFile("redump.zip", "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) 35 | 36 | # clrmamepro XML file 37 | tag_clrmamepro = ET.Element("clrmamepro") 38 | 39 | for dat in dat_list: 40 | print(f"Downloading {dat}") 41 | # section for this dat in the XML file 42 | tag_datfile = ET.SubElement(tag_clrmamepro, "datfile") 43 | 44 | response = requests.get(URL_HOME + "datfile/" + dat, timeout=150) 45 | content_header = response.headers["Content-Disposition"] 46 | 47 | # XML version 48 | dat_date = re.findall(regex["date"], content_header)[0] 49 | ET.SubElement(tag_datfile, "version").text = dat_date 50 | 51 | # XML name & description 52 | temp_name = re.findall(regex["name"], content_header)[0] 53 | # trim the - from the end (if exists) 54 | if temp_name.endswith("-"): 55 | temp_name = temp_name[:-2] 56 | elif temp_name.endswith("BIOS"): 57 | temp_name = temp_name + " Images" 58 | ET.SubElement(tag_datfile, "name").text = temp_name 59 | ET.SubElement(tag_datfile, "description").text = temp_name 60 | 61 | # URL tag in XML 62 | ET.SubElement(tag_datfile, "url").text = XML_URL 63 | 64 | # File tag in XML 65 | original_filename = re.findall(regex["filename"], content_header)[0] 66 | filename = f"{original_filename[:-4]}.dat" 67 | ET.SubElement(tag_datfile, "file").text = filename 68 | 69 | # Author tag in XML 70 | ET.SubElement(tag_datfile, "author").text = "redump.org" 71 | 72 | # Command XML tag 73 | ET.SubElement(tag_datfile, "comment").text = "_" 74 | 75 | # Get the DAT file 76 | datfile_name = f"{filename[:-4]}.dat" 77 | print(f"DAT filename: {datfile_name}") 78 | if original_filename.endswith(".zip"): 79 | # extract datfile from zip to store in the DB zip 80 | zipdata = BytesIO() 81 | zipdata.write(response.content) 82 | archive = zipfile.ZipFile(zipdata) 83 | zip_object.writestr(datfile_name, archive.read(datfile_name)) 84 | else: 85 | # add datfile to DB zip file 86 | datfile = response.text 87 | zip_object.writestr(datfile_name, datfile) 88 | print() 89 | sleep(5) 90 | 91 | # store clrmamepro XML file 92 | xmldata = ET.tostring(tag_clrmamepro).decode() 93 | 94 | with open(XML_FILENAME, "w", encoding="utf-8") as xmlfile: 95 | xmlfile.write(xmldata) 96 | 97 | print("Finished") 98 | 99 | 100 | try: 101 | update_XML() 102 | except KeyboardInterrupt: 103 | pass 104 | -------------------------------------------------------------------------------- /smdb.py: -------------------------------------------------------------------------------- 1 | import zipfile 2 | import os 3 | import shutil 4 | import subprocess 5 | from datetime import date, datetime 6 | from io import StringIO, TextIOWrapper 7 | from xml.dom import minidom 8 | from io import BytesIO 9 | from time import sleep 10 | import xml.etree.ElementTree as ET 11 | from lxml import etree, objectify 12 | from lxml import etree 13 | import re 14 | import requests 15 | from collections import defaultdict 16 | 17 | 18 | clr_root = etree.Element('clrmamepro') 19 | 20 | 21 | def generate(txt, platform, txt_date, zf): 22 | txt_date = txt_date.strftime("%Y-%m-%d %H:%M:%S") 23 | games = defaultdict(list) 24 | 25 | dtd_file = StringIO(requests.get("http://www.logiqx.com/Dats/datafile.dtd").text) 26 | dtd = etree.DTD(dtd_file) 27 | 28 | # Get Saturn supplement 29 | for line in txt: 30 | # Some have a sixth columen with the rom size 31 | sha256, name, sha1, md5, crc32 = line.split('\t')[:5] 32 | 33 | rom_name = name[name.find('/', name.find('/') + 1) + 1:].replace('/', '\\') 34 | name = name[name.find('/') + 1:name.find('/', name.find('/') + 1)].replace('/', '\\') 35 | games[name].append((rom_name, sha1, md5, crc32)) 36 | 37 | root = etree.Element('datafile') 38 | header = etree.SubElement(root, 'header') 39 | name = etree.SubElement(header, 'name') 40 | name.text = platform 41 | description = etree.SubElement(header, 'description') 42 | description.text = f'{platform} ({txt_date})' 43 | version = etree.SubElement(header, 'version') 44 | version.text = str(txt_date) 45 | date_xml = etree.SubElement(header, 'date') 46 | date_xml.text = str(txt_date) 47 | author = etree.SubElement(header, 'author') 48 | author.text = 'SmokeMonster' 49 | homepage = etree.SubElement(header, 'homepage') 50 | homepage.text = 'https://github.com/SmokeMonsterPacks/EverDrive-Packs-Lists-Database' 51 | url = etree.SubElement(header, 'url') 52 | url.text = 'https://github.com/SmokeMonsterPacks/EverDrive-Packs-Lists-Database' 53 | 54 | for game_name, roms in games.items(): 55 | game = etree.SubElement(root, 'game') 56 | game.set('name', game_name) 57 | category = etree.SubElement(game, 'category') 58 | category.text = 'Games' 59 | description = etree.SubElement(game, 'description') 60 | description.text = game_name 61 | for props in roms: 62 | rom = etree.SubElement(game, 'rom') 63 | rom.set('name', props[0]) 64 | rom.set('crc', props[3]) 65 | rom.set('md5', props[2]) 66 | rom.set('sha1', props[1]) 67 | 68 | etree_str = (etree.tostring(root, encoding="UTF-8", 69 | xml_declaration=True, 70 | doctype='')) 71 | final_xml = minidom.parseString(etree_str).toprettyxml(indent=" ") 72 | zf.writestr(f'datfile_{platform.replace(" ", "_").lower()}.dat', final_xml) 73 | 74 | # clrmamepro xml 75 | df = etree.SubElement(clr_root, 'datfile') 76 | version = etree.SubElement(df, 'version') 77 | version.text = str(txt_date) 78 | name = etree.SubElement(df, 'name') 79 | name.text = platform 80 | description = etree.SubElement(df, 'description') 81 | description.text = platform 82 | url_xml = etree.SubElement(df, 'url') 83 | url_xml.text = 'https://github.com/hugo19941994/auto-datfile-generator/releases/latest/download/smdb.zip' 84 | file_xml = etree.SubElement(df, 'file') 85 | file_xml.text = f'datfile_{platform.replace(" ", "_").lower()}.dat' 86 | author_xml = etree.SubElement(df, 'author') 87 | author_xml.text = 'SmokeMonster' 88 | comment_xml = etree.SubElement(df, 'comment') 89 | comment_xml.text = '_' 90 | 91 | # Multiple roms in a game is technically invalid, but clrmamepro accepts it 92 | # print(dtd.validate(root)) 93 | # print(dtd.error_log.filter_from_errors()[0]) 94 | 95 | """ 96 | 97 | r = requests.get('https://github.com/SmokeMonsterPacks/EverDrive-Packs-Lists-Database/archive/master.zip') 98 | zipdata = BytesIO() 99 | zipdata.write(r.content) 100 | archive = zipfile.ZipFile(zipdata) 101 | 102 | for f in archive.namelist(): 103 | if f.startswith('EverDrive-Packs-Lists-Database-master/EverDrive Pack SMDBs/') and f.endswith('.txt'): 104 | file_date = archive.getinfo(f).date_time 105 | print(file_date) 106 | name = f.replace('EverDrive-Packs-Lists-Database-master/EverDrive Pack SMDBs/', '').replace('.txt', '') 107 | with archive.open(f, 'r') as txt: 108 | txt_file = TextIOWrapper(txt) 109 | generate(txt_file, name, zf) 110 | 111 | 112 | """ 113 | 114 | # git log -1 --pretty="format:%ci" 115 | 116 | zf = zipfile.ZipFile('smdb.zip', 'w') 117 | 118 | shutil.rmtree('./EverDrive-Packs-Lists-Database', ignore_errors=True, onerror=None) 119 | subprocess.call('git clone https://github.com/SmokeMonsterPacks/EverDrive-Packs-Lists-Database.git', shell = True) 120 | folder = './EverDrive-Packs-Lists-Database/EverDrive Pack SMDBs' 121 | for f in os.listdir(folder): 122 | file_date = subprocess.run(['git','--no-pager', 'log', '-1', '--pretty="format:%ci"', f], capture_output=True, cwd=folder).stdout.decode('utf-8')[8:-2] 123 | txt_date = datetime.strptime(file_date,'%Y-%m-%d %H:%M:%S %z') 124 | if '.txt' not in f: 125 | # manual packs in 7z 126 | continue 127 | name = f.replace('.txt', '') 128 | with open(f'{folder}/{f}') as txt: 129 | generate(txt, name, txt_date, zf) 130 | 131 | zf.close() 132 | 133 | final_xml = minidom.parseString(etree.tostring(clr_root)).toprettyxml(indent=" ") 134 | with open(f'smdb.xml', 'w') as f: 135 | f.write(final_xml) 136 | --------------------------------------------------------------------------------