├── .gitignore ├── Fingerprinter.jar ├── README.md ├── fingerprints └── netflix_fingerprints.txt ├── runFingerprinter.bash └── src ├── Encoding.java └── Fingerprinter.java /.gitignore: -------------------------------------------------------------------------------- 1 | _site/* 2 | _theme_packages/* 3 | 4 | Thumbs.db 5 | .DS_Store 6 | 7 | !.gitkeep 8 | 9 | *~ 10 | *.swp 11 | *.class 12 | 13 | .rbenv-version 14 | .rvmrc 15 | -------------------------------------------------------------------------------- /Fingerprinter.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andrewreed/Fingerprinter/f731305110bc70e0abfa285d63d6c932c434b141/Fingerprinter.jar -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## _Fingerprinter_ 2 | 3 | A utility to create fingerprints of Netflix videos that can be used in _dashid_ 4 | to identify specific videos in anonymized, header-only network traces. 5 | 6 | _Fingerprinter_ downloads the header of each encoding's .ismv, which are then used to calculate 7 | the sequence of segment sizes for each encoding. 8 | 9 | ### Instructions 10 | 11 | 1. Create a directory structure of the form: 12 | 13 | netflix/movie/{movie name}/ 14 | netflix/show/{series name}/{season number}/{episode number}/ 15 | 16 | At this point, each folder should be empty. 17 | 2. Install the Tamper Data plugin for Firefox. 18 | 3. Open Firefox and go to Netflix. 19 | 4. Open the Tamper Data plugin. 20 | 5. Play a Netflix video. 21 | 6. Once the video has started to stream, right click inside the upper window of Tamper Data and select _Export XML - All_. 22 | 7. Save the file as _capture.xml_. 23 | 8. Copy _capture.xml_ to the appropriate video subfolder created in Step 1. 24 | 9. Repeat steps 5-8 for several videos. Do not spend too much time on this task, 25 | as each playback of a video will timeout after some time, at which point the URLs will become invalid. 26 | 10. From within the directory that contains the __netflix__ directory, run the following command: 27 | 28 | ./runFingerprinter.bash >> netflix_fingerprints.txt 29 | 30 | This script will parse the XML files and create text files in each video subfolder that list the URLs needed by Fingerprinter. 31 | The script will then run Fingerprinter and provide it with the list of subfolders via stdin. 32 | 11. Once you have run Fingerprinter in Step 10, you should verify that fingerprints were produced correctly. If so, you should delete all 33 | folders inside __netflix/__ before creating additional fingerprints. 34 | 35 | ### Credit / Copying 36 | 37 | As a work of the United States Government, _Fingerprinter_ is 38 | in the public domain within the United States. Additionally, 39 | Andrew Reed waives copyright and related rights in the work 40 | worldwide through the CC0 1.0 Universal public domain dedication 41 | (which can be found at http://creativecommons.org/publicdomain/zero/1.0/). 42 | -------------------------------------------------------------------------------- /runFingerprinter.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find "netflix" -type f | grep ".xml" | while read line 3 | do 4 | path=`dirname $line` 5 | grep "range/0-" $line | cut -d "\"" -f2 | sort | uniq > $path"/urls.txt" 6 | done 7 | find "netflix" -type d -links 2 | java -jar Fingerprinter.jar 8 | -------------------------------------------------------------------------------- /src/Encoding.java: -------------------------------------------------------------------------------- 1 | import java.io.*; 2 | import java.util.*; 3 | import java.net.*; 4 | 5 | public class Encoding { 6 | // HTTP headers for a valid GET request to Netflix 7 | private static final String ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; 8 | private static final String ACCEPT_ENCODING = "gzip, deflate"; 9 | private static final String ACCEPT_LANGUAGE = "en-US,en;q=0.5"; 10 | private static final String CONNECTION = "keep-alive"; 11 | private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; rv:27.0) Gecko/20100101 Firefox/27.0"; 12 | 13 | private String host; 14 | private double avgBitrate; 15 | private int[] segmentSizes; 16 | private byte[] header; 17 | 18 | // Constructor 19 | public Encoding(String urlString) { 20 | this.host = urlString.split("/")[2]; 21 | 22 | getHeader(urlString); 23 | 24 | calculateSegmentSizes(); 25 | 26 | double totalSum = 0.0; 27 | for (int i = 0; i < segmentSizes.length; i++) { 28 | totalSum += segmentSizes[i]; 29 | } 30 | 31 | this.avgBitrate = (((totalSum / segmentSizes.length) * 8.0) / 4.0) / 1000.0; 32 | 33 | } 34 | 35 | public double getAvgBitrate() { 36 | return avgBitrate; 37 | } 38 | 39 | public int[] getSegmentSizes() { 40 | return segmentSizes; 41 | } 42 | 43 | private void getHeader(String urlString) { 44 | URL url; 45 | HttpURLConnection conn; 46 | InputStream is; 47 | 48 | try { 49 | url = new URL(urlString); 50 | 51 | conn = (HttpURLConnection) url.openConnection(); 52 | conn.setRequestMethod("GET"); 53 | conn.setRequestProperty("Accept", ACCEPT); 54 | conn.setRequestProperty("Accept-Encoding", ACCEPT_ENCODING); 55 | conn.setRequestProperty("Accept-Language", ACCEPT_LANGUAGE); 56 | conn.setRequestProperty("Connection", CONNECTION); 57 | conn.setRequestProperty("Host", host); 58 | conn.setRequestProperty("User-Agent", USER_AGENT); 59 | 60 | is = conn.getInputStream(); 61 | 62 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 63 | byte[] buffer = new byte[4096]; 64 | for (int count; ((count = is.read(buffer)) != -1); ) { 65 | baos.write(buffer, 0, count); 66 | } 67 | baos.close(); 68 | 69 | header = baos.toByteArray(); 70 | 71 | is.close(); 72 | } catch (Exception e) { 73 | System.out.println("ERROR: Unable to download header."); 74 | System.exit(0); 75 | } 76 | } 77 | 78 | private void calculateSegmentSizes() { 79 | int sidxLoc = 0; 80 | 81 | for (int i = 0; i < header.length-3; i++) { 82 | 83 | char c1 = (char) (header[i] & 0xFF); 84 | char c2 = (char) (header[i+1] & 0xFF); 85 | char c3 = (char) (header[i+2] & 0xFF); 86 | char c4 = (char) (header[i+3] & 0xFF); 87 | 88 | if ((c1 == 's') && (c2 == 'i') && (c3 == 'd') && (c4 == 'x')) { 89 | sidxLoc = i; 90 | break; 91 | } 92 | } 93 | 94 | if (sidxLoc == 0) { 95 | System.out.println("ERROR: Could not find segment index box."); 96 | System.exit(0); 97 | } 98 | 99 | int refCountLoc = sidxLoc + 34; 100 | 101 | int refCount = 0; 102 | refCount = (header[refCountLoc] & 0xff); 103 | refCount = (refCount << 8) + (header[refCountLoc+1] & 0xff); 104 | 105 | segmentSizes = new int[(int)Math.ceil(refCount / 2.0)]; 106 | 107 | int segmentSizeLoc = refCountLoc + 2; 108 | 109 | for (int i = 0; i < refCount; i++) { 110 | int currentSize = 0; 111 | currentSize = (header[segmentSizeLoc] & 0xff); 112 | currentSize = (currentSize << 8) + (header[segmentSizeLoc+1] & 0xff); 113 | currentSize = (currentSize << 8) + (header[segmentSizeLoc+2] & 0xff); 114 | currentSize = (currentSize << 8) + (header[segmentSizeLoc+3] & 0xff); 115 | 116 | segmentSizes[i/2] += currentSize; 117 | 118 | segmentSizeLoc += 12; 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/Fingerprinter.java: -------------------------------------------------------------------------------- 1 | import java.util.*; 2 | import java.math.*; 3 | import java.io.*; 4 | import java.net.*; 5 | 6 | public class Fingerprinter { 7 | 8 | private static final int[] BITRATES = {235,375,560,750,1050,1750,2350,3000}; 9 | 10 | public static void main(String args[]) { 11 | 12 | Scanner sc = new Scanner(System.in); 13 | 14 | while (sc.hasNextLine()) { 15 | String pathToURLs = sc.nextLine(); 16 | 17 | List encodings = retrieveEncodings(pathToURLs); 18 | 19 | System.out.print(pathToURLs + "\t"); 20 | 21 | int[][] segmentSizes = new int[encodings.size()][]; 22 | 23 | int count = 0; 24 | 25 | for (Encoding encoding : encodings) { 26 | segmentSizes[count] = encoding.getSegmentSizes(); 27 | 28 | System.out.print(BITRATES[count]); 29 | 30 | count++; 31 | 32 | if (count < encodings.size()) { 33 | System.out.print(","); 34 | } else { 35 | System.out.print("\t"); 36 | } 37 | } 38 | 39 | int numSegments = segmentSizes[0].length; 40 | 41 | for (int j = 0; j < numSegments; j++) { 42 | for (int i = 0; i < count; i++) { 43 | System.out.print(segmentSizes[i][j]); 44 | 45 | if ((i == (count-1)) && (j == (numSegments-1))) { 46 | System.out.print("\n"); 47 | } else { 48 | System.out.print(","); 49 | } 50 | } 51 | } 52 | } 53 | 54 | sc.close(); 55 | } 56 | 57 | private static List retrieveEncodings(String pathToURLs) { 58 | List encodings = new LinkedList(); 59 | 60 | try { 61 | File urlList = new File(pathToURLs + "/urls.txt"); 62 | 63 | Scanner sc = new Scanner(urlList); 64 | 65 | while (sc.hasNextLine()) { 66 | Encoding encoding = new Encoding(URLDecoder.decode(sc.nextLine(),"UTF-8")); 67 | 68 | if (encoding.getAvgBitrate() > 100) { 69 | encodings.add(encoding); 70 | } 71 | } 72 | 73 | sc.close(); 74 | 75 | } catch (Exception e) { 76 | System.out.println("ERROR: Unable to retrieve encodings."); 77 | System.exit(0); 78 | } 79 | 80 | Encoding[] sortingArray = encodings.toArray(new Encoding[encodings.size()]); 81 | 82 | for (int j = 1; j < sortingArray.length; j++) { 83 | Encoding key = sortingArray[j]; 84 | int i = j - 1; 85 | while ((i > -1) && (sortingArray[i].getAvgBitrate() > key.getAvgBitrate())) { 86 | sortingArray[i+1] = sortingArray[i]; 87 | i--; 88 | } 89 | sortingArray[i+1] = key; 90 | } 91 | 92 | List sortedEncodings = new LinkedList(); 93 | for (int i=0; i < sortingArray.length; i++) { 94 | sortedEncodings.add(sortingArray[i]); 95 | } 96 | return sortedEncodings; 97 | } 98 | } 99 | --------------------------------------------------------------------------------