2 | 
3 |
113 |
118 | );
119 | }
120 |
121 | return (
122 | 114 | This application is only intended for desktop use with a mouse or 115 | trackpad. 116 |
117 |
123 |
180 | );
181 | }
182 |
--------------------------------------------------------------------------------
/puppeteer-scribe/server/server.py:
--------------------------------------------------------------------------------
1 | import os
2 | import numpy as np
3 | import torch
4 | import torch.nn as nn
5 | from flask import Flask, request, jsonify
6 | import json
7 | import matplotlib
8 | matplotlib.use('Agg')
9 | import matplotlib.pyplot as plt
10 | import io
11 | import base64
12 | import glob
13 |
14 | device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
15 |
16 | SCREEN_WIDTH = 1920
17 | SCREEN_HEIGHT = 1080
18 |
19 | class CVAE(nn.Module):
20 | def __init__(self, input_size=2, condition_size=4, hidden_size=128, latent_size=32):
21 | super(CVAE, self).__init__()
22 | self.latent_size = latent_size
23 | self.input_size = input_size # To access input_size in methods
24 |
25 | # Encoder
26 | self.encoder_lstm = nn.LSTM(input_size + 1, hidden_size, batch_first=True) # +1 for time intervals
27 | self.fc_mu = nn.Linear(hidden_size + condition_size, latent_size)
28 | self.fc_logvar = nn.Linear(hidden_size + condition_size, latent_size)
29 |
30 | # Decoder
31 | # Input size: input_size + latent_size + condition_size + 1
32 | self.decoder_lstm = nn.LSTM(input_size + latent_size + condition_size + 1, hidden_size, batch_first=True)
33 | self.output_layer = nn.Linear(hidden_size, input_size)
34 |
35 | def decode(self, z, condition, seq_len, start_point):
36 | batch_size = z.size(0)
37 | outputs = []
38 | hidden = None
39 |
40 | # Initialize x_t with the start point
41 | x_t = start_point.to(z.device) # Shape: (batch_size, input_size)
42 |
43 | # Use average time intervals or a constant value
44 | average_delta_time = 0.05 # Adjust based on your data (in seconds)
45 | time_intervals = torch.full((batch_size, seq_len), average_delta_time).to(z.device)
46 |
47 | for t in range(seq_len):
48 | delta_t = time_intervals[:, t].unsqueeze(1) # Shape: (batch_size, 1)
49 | z_t = z # Shape: (batch_size, latent_size)
50 | cond_t = condition # Shape: (batch_size, condition_size)
51 |
52 | # Include x_t in the decoder input
53 | decoder_input = torch.cat([x_t, z_t, cond_t, delta_t], dim=-1).unsqueeze(1)
54 | # Shape: (batch_size, 1, input_size + latent_size + condition_size + 1)
55 |
56 | output, hidden = self.decoder_lstm(decoder_input, hidden)
57 | x_t = self.output_layer(output.squeeze(1)) # Shape: (batch_size, input_size)
58 | outputs.append(x_t.unsqueeze(1))
59 |
60 | # x_t is used in the next iteration
61 |
62 | outputs = torch.cat(outputs, dim=1) # Shape: (batch_size, seq_len, input_size)
63 | return outputs
64 |
65 | def generate_path(model, start_point, end_point, seq_len=50):
66 | model.eval()
67 | with torch.no_grad():
68 | condition = torch.cat([start_point, end_point], dim=-1).unsqueeze(0).to(device) # Shape: (1, 4)
69 | z = torch.randn(1, model.latent_size).to(device) # Sample from standard normal
70 |
71 | generated_seq = model.decode(z, condition, seq_len, start_point.unsqueeze(0))
72 | generated_seq = generated_seq.squeeze(0).cpu().numpy() # Shape: (seq_len, input_size)
73 | return generated_seq
74 |
75 | def transform_path_to_endpoints(path, start_point, end_point):
76 | """
77 | Transform the generated path so that it starts at start_point and ends at end_point.
78 | This is done by applying an affine transformation (rotation, scaling, and translation).
79 | """
80 | original_start = path[0]
81 | original_end = path[-1]
82 |
83 | # Vectors from start to end
84 | v_o = original_end - original_start
85 | v_d = end_point - start_point
86 |
87 | # Lengths of the vectors
88 | len_o = np.linalg.norm(v_o)
89 | len_d = np.linalg.norm(v_d)
90 |
91 | # Avoid division by zero
92 | if len_o == 0 or len_d == 0:
93 | # Return a straight line from start_point to end_point
94 | transformed_path = np.linspace(start_point, end_point, num=len(path))
95 | return transformed_path
96 |
97 | # Scale factor
98 | scale = len_d / len_o
99 |
100 | # Compute the angle between v_o and v_d
101 | # First, normalize the vectors
102 | v_o_unit = v_o / len_o
103 | v_d_unit = v_d / len_d
104 |
105 | # Compute rotation angle
106 | angle = np.arctan2(v_d_unit[1], v_d_unit[0]) - np.arctan2(v_o_unit[1], v_o_unit[0])
107 |
108 | # Build rotation matrix
109 | cos_theta = np.cos(angle)
110 | sin_theta = np.sin(angle)
111 | R = np.array([[cos_theta, -sin_theta],
112 | [sin_theta, cos_theta]])
113 |
114 | # Apply transformation to the path
115 | # Shift the path to origin based on original_start
116 | shifted_path = path - original_start
117 |
118 | # Rotate
119 | rotated_path = shifted_path @ R.T # Using matrix multiplication
120 |
121 | # Scale
122 | scaled_path = rotated_path * scale
123 |
124 | # Translate to desired start point
125 | transformed_path = scaled_path + start_point
126 |
127 | return transformed_path
128 |
129 | app = Flask(__name__)
130 |
131 | # Assuming script is being ran from parent directory
132 | script_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
133 | model_dir = os.path.join(script_dir, 'model')
134 |
135 | model_files = glob.glob(os.path.join(model_dir, '*.pt'))
136 |
137 | if not model_files:
138 | raise FileNotFoundError("No model files found in the 'model' directory.")
139 |
140 | MODEL_FILE = max(model_files, key=os.path.getmtime)
141 |
142 | # Initialize the model architecture
143 | model = CVAE(input_size=2, condition_size=4, hidden_size=128, latent_size=32).to(device)
144 |
145 | # Load the saved model parameters
146 | if not os.path.exists(MODEL_FILE):
147 | raise FileNotFoundError(f"Model file {MODEL_FILE} not found. Please check the path.")
148 |
149 | model.load_state_dict(torch.load(MODEL_FILE, map_location=device))
150 | print(f"Model loaded from {MODEL_FILE}")
151 |
152 | @app.route('/generate_path', methods=['GET'])
153 | def api_generate_path():
154 | start_point = request.args.get('start_point', default=None)
155 | end_point = request.args.get('end_point', default=None)
156 | visualize = request.args.get('visualize', default='false')
157 |
158 | if not start_point or not end_point:
159 | return jsonify({'error': 'Both start and end points are required.'}), 400
160 |
161 | try:
162 | start_point = list(map(float, start_point.split(',')))
163 | end_point = list(map(float, end_point.split(',')))
164 |
165 | if not (len(start_point) == 2 and len(end_point) == 2):
166 | return jsonify({'error': 'Start and end points must have two coordinates [x, y].'}), 400
167 |
168 | # Normalize the coordinates
169 | start_point_normalized = np.array([
170 | start_point[0] / SCREEN_WIDTH,
171 | start_point[1] / SCREEN_HEIGHT
172 | ], dtype=np.float32)
173 | end_point_normalized = np.array([
174 | end_point[0] / SCREEN_WIDTH,
175 | end_point[1] / SCREEN_HEIGHT
176 | ], dtype=np.float32)
177 |
178 | start_point_tensor = torch.tensor(start_point_normalized).to(torch.float32).to(device)
179 | end_point_tensor = torch.tensor(end_point_normalized).to(torch.float32).to(device)
180 |
181 | # Generate the mouse movement path
182 | generated_path = generate_path(model, start_point_tensor, end_point_tensor, seq_len=50)
183 |
184 | # Transform the path to match the exact input start and end points
185 | transformed_path = transform_path_to_endpoints(generated_path, start_point_normalized, end_point_normalized)
186 |
187 | # Denormalize the transformed path back to screen coordinates
188 | transformed_path_denormalized = transformed_path * np.array([SCREEN_WIDTH, SCREEN_HEIGHT])
189 |
190 | # Convert the path to a list of [x, y] coordinates
191 | path_list = transformed_path_denormalized.tolist()
192 |
193 | response_data = {'path': path_list}
194 |
195 | # If visualization is requested
196 | if visualize.lower() == 'true':
197 | # Generate the plot
198 | plt.figure(figsize=(6, 6))
199 |
200 | # Plot the transformed path
201 | plt.plot(transformed_path_denormalized[:, 0], transformed_path_denormalized[:, 1],
202 | marker='o', label='Transformed Generated Path')
203 |
204 | # Plot start and end points
205 | plt.scatter(start_point[0], start_point[1],
206 | color='green', label='Start Point (Input)', s=100, zorder=5)
207 | plt.scatter(end_point[0], end_point[1],
208 | color='red', label='End Point (Input)', s=100, zorder=5)
209 |
210 | # Display coordinates
211 | plt.text(start_point[0], start_point[1] + 20,
212 | f"({start_point[0]:.1f}, {start_point[1]:.1f})",
213 | color='green')
214 | plt.text(end_point[0], end_point[1] + 20,
215 | f"({end_point[0]:.1f}, {end_point[1]:.1f})",
216 | color='red')
217 |
218 | plt.legend()
219 | plt.title('Transformed Generated Path')
220 | plt.xlabel('X Coordinate')
221 | plt.ylabel('Y Coordinate')
222 | plt.xlim(0, SCREEN_WIDTH)
223 | plt.ylim(0, SCREEN_HEIGHT)
224 | plt.gca().invert_yaxis()
225 | plt.grid(True)
226 |
227 | # Save the plot to a bytes buffer
228 | buf = io.BytesIO()
229 | plt.savefig(buf, format='png')
230 | buf.seek(0)
231 | plt.close()
232 |
233 | # Encode the image in base64
234 | img_base64 = base64.b64encode(buf.getvalue()).decode('utf-8')
235 |
236 | # Add the image to the response
237 | response_data['plot'] = img_base64
238 |
239 | return jsonify(response_data), 200
240 |
241 | except Exception as e:
242 | print(f"Error: {e}")
243 | return jsonify({'error': str(e)}), 500
244 |
245 | if __name__ == '__main__':
246 | app.run(host='0.0.0.0', port=3000)
--------------------------------------------------------------------------------
/puppeteer-scribe/index.ts:
--------------------------------------------------------------------------------
1 | import axios from "axios";
2 | import { Frame, KeyInput, Page } from "puppeteer";
3 | import { setTimeout } from "timers/promises";
4 |
5 | interface Point {
6 | x: number;
7 | y: number;
8 | }
9 |
10 | interface MoveMouseOptions {
11 | visualize?: boolean;
12 | log?: boolean;
13 | }
14 |
15 | interface ScribeOptions {
16 | hesitationDelay: number;
17 | clickDelay: number;
18 | moveDelay: number;
19 | apiUrl: string;
20 | visualize: boolean;
21 | startPosition: Point;
22 | wpm: number;
23 | }
24 |
25 | interface TypeOptions {
26 | wpm?: number;
27 | }
28 |
29 | interface ClickOptions {
30 | frame?: Frame;
31 | }
32 |
33 | export default class Scribe {
34 | page: Page;
35 | options: ScribeOptions;
36 | mouse: Point;
37 |
38 | constructor(page: Page, options: PartialProject Scribe
124 |
129 | {boxes.map((box) => (
130 |
163 |
170 | handleBoxClick(box.id)}
143 | >
144 | {box.id}
145 |
146 | ))}
147 | {isTracking && (
148 |
161 | )}
162 |
175 | {currentBox === "A" && "Click point A to start tracking"}
176 | {currentBox === "B" && "Now click point B to finish"}
177 | {!currentBox && pathData && "Path recorded! Check the console for data"}
178 |
179 |