View original post on Mastodon
184 | {{ end }} 185 | ``` 186 | 187 | ## Development 188 | 189 | ```bash 190 | # Download dependencies 191 | make deps 192 | 193 | # Build 194 | make build 195 | 196 | # Clean 197 | make clean 198 | 199 | # Help 200 | make help 201 | ``` 202 | 203 | ## Requirements 204 | 205 | - Go 1.21 or later 206 | - Internet connection for fetching posts and media 207 | - Valid Mastodon access token 208 | 209 | ## Notes 210 | 211 | - Only fetches **public** posts 212 | - Skips reblogs/boosts (fetches original content only) 213 | - Media files are downloaded as-is (no processing/resizing) 214 | - Existing files are not re-downloaded (basic deduplication) 215 | - HTML tags are stripped from post content while preserving line breaks 216 | - Post content is stored in markdown body, not in YAML front matter 217 | - When using `--max-status-id`, the tool fetches ALL posts chronologically up to that status ID (useful for incremental syncing) 218 | - The `--ignore-replies` flag skips posts that are replies to other users' posts, but keeps your own threads (replies to your own posts) 219 | - The `--skip-existing` flag checks for existing markdown files and skips processing them, making subsequent runs much faster 220 | - The `--trailing-tag` flag filters posts by their ending tag (case insensitive) and removes the tag from the final markdown output -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "html" 8 | "io" 9 | "log" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "path" 14 | "path/filepath" 15 | "regexp" 16 | "strings" 17 | "time" 18 | 19 | "github.com/mattn/go-mastodon" 20 | "gopkg.in/yaml.v3" 21 | ) 22 | 23 | type Config struct { 24 | Instance string 25 | User string 26 | Token string 27 | ContentDir string 28 | MediaDir string 29 | MaxStatusID string 30 | IgnoreReplies bool 31 | SkipExisting bool 32 | TrailingTag string 33 | } 34 | 35 | type PostMetadata struct { 36 | Date time.Time `yaml:"date"` 37 | OriginalURL string `yaml:"original_url"` 38 | Media []string `yaml:"media,omitempty"` 39 | } 40 | 41 | func main() { 42 | var config Config 43 | 44 | // CLI flags 45 | flag.StringVar(&config.Instance, "instance", "mastodon.social", "Mastodon instance URL") 46 | flag.StringVar(&config.User, "user", "", "Mastodon username") 47 | flag.StringVar(&config.ContentDir, "content-dir", "./content", "Directory for markdown files") 48 | flag.StringVar(&config.MediaDir, "media-dir", "./content/attachments", "Directory for media files") 49 | flag.StringVar(&config.MaxStatusID, "max-status-id", "", "Maximum status ID to fetch") 50 | flag.BoolVar(&config.IgnoreReplies, "ignore-replies", false, "Ignore replies to other people") 51 | flag.BoolVar(&config.SkipExisting, "skip-existing", false, "Skip posts that already exist as markdown files") 52 | flag.StringVar(&config.TrailingTag, "trailing-tag", "", "Specific trailing tag to filter for (e.g., '#BLOG' or '^PUBLISH')") 53 | flag.Parse() 54 | 55 | // Get token from environment 56 | config.Token = os.Getenv("MASTODON_TOKEN") 57 | if config.Token == "" { 58 | log.Fatal("MASTODON_TOKEN environment variable is required") 59 | } 60 | 61 | // Validate required parameters 62 | if config.User == "" { 63 | log.Fatal("--user parameter is required") 64 | } 65 | 66 | // Ensure directories exist 67 | if err := os.MkdirAll(config.ContentDir, 0755); err != nil { 68 | log.Fatalf("Failed to create content directory: %v", err) 69 | } 70 | if err := os.MkdirAll(config.MediaDir, 0755); err != nil { 71 | log.Fatalf("Failed to create media directory: %v", err) 72 | } 73 | 74 | // Create Mastodon client 75 | client := mastodon.NewClient(&mastodon.Config{ 76 | Server: "https://" + config.Instance, 77 | AccessToken: config.Token, 78 | }) 79 | 80 | // Verify credentials and get account info 81 | account, err := client.GetAccountCurrentUser(context.Background()) 82 | if err != nil { 83 | log.Fatalf("Failed to verify credentials: %v", err) 84 | } 85 | 86 | fmt.Printf("Authenticated as: @%s\n", account.Username) 87 | 88 | // Find the target user account 89 | accounts, err := client.AccountsSearch(context.Background(), config.User, 1) 90 | if err != nil { 91 | log.Fatalf("Failed to search for user: %v", err) 92 | } 93 | if len(accounts) == 0 { 94 | log.Fatalf("User @%s not found", config.User) 95 | } 96 | 97 | targetAccount := accounts[0] 98 | fmt.Printf("Found user: @%s (%s)\n", targetAccount.Username, targetAccount.DisplayName) 99 | 100 | // Fetch user's statuses (public only) 101 | var pg mastodon.Pagination 102 | allStatuses := []*mastodon.Status{} 103 | reachedMaxStatusID := false 104 | 105 | fmt.Printf("Fetching statuses") 106 | if config.MaxStatusID != "" { 107 | fmt.Printf(" up to status ID: %s", config.MaxStatusID) 108 | } 109 | if config.IgnoreReplies { 110 | fmt.Printf(" (ignoring replies)") 111 | } 112 | fmt.Println() 113 | 114 | for { 115 | statuses, err := client.GetAccountStatuses(context.Background(), targetAccount.ID, &pg) 116 | if err != nil { 117 | log.Fatalf("Failed to fetch statuses: %v", err) 118 | } 119 | if len(statuses) == 0 { 120 | break 121 | } 122 | 123 | // Process statuses and apply filters 124 | for _, status := range statuses { 125 | // Check if we've reached the maximum status ID 126 | if config.MaxStatusID != "" && string(status.ID) == config.MaxStatusID { 127 | fmt.Printf("Reached maximum status ID: %s\n", config.MaxStatusID) 128 | reachedMaxStatusID = true 129 | break 130 | } 131 | 132 | // Filter for public posts only 133 | if status.Visibility != "public" { 134 | continue 135 | } 136 | 137 | // Skip replies to other people if ignore-replies flag is set 138 | // Keep replies to self (threads) but ignore replies to others 139 | if config.IgnoreReplies && status.InReplyToID != nil && status.InReplyToAccountID != nil { 140 | // Convert InReplyToAccountID to string for comparison 141 | replyToAccountID := "" 142 | if accountID, ok := status.InReplyToAccountID.(string); ok { 143 | replyToAccountID = accountID 144 | } else if accountID, ok := status.InReplyToAccountID.(mastodon.ID); ok { 145 | replyToAccountID = string(accountID) 146 | } 147 | 148 | // Skip if replying to someone else (not self) 149 | if replyToAccountID != string(targetAccount.ID) { 150 | continue 151 | } 152 | } 153 | 154 | // Skip posts without specified trailing tag if trailing-tag is set 155 | if config.TrailingTag != "" && !hasTrailingTag(status.Content, config.TrailingTag) { 156 | continue 157 | } 158 | 159 | allStatuses = append(allStatuses, status) 160 | } 161 | 162 | // Stop if we reached the max status ID 163 | if reachedMaxStatusID { 164 | break 165 | } 166 | 167 | // Check if there are more pages 168 | if pg.MaxID == "" { 169 | break 170 | } 171 | } 172 | 173 | fmt.Printf("Found %d posts to process", len(allStatuses)) 174 | if config.IgnoreReplies { 175 | fmt.Printf(" (replies excluded)") 176 | } 177 | if config.SkipExisting { 178 | fmt.Printf(" (skipping existing)") 179 | } 180 | if config.TrailingTag != "" { 181 | fmt.Printf(" (filtering by: %s)", config.TrailingTag) 182 | } 183 | fmt.Println() 184 | 185 | // Process each status 186 | processedCount := 0 187 | skippedCount := 0 188 | 189 | for i, status := range allStatuses { 190 | fmt.Printf("Processing post %d/%d (ID: %s)...", i+1, len(allStatuses), status.ID) 191 | 192 | // Check if file already exists when skip-existing flag is set 193 | if config.SkipExisting { 194 | // Parse the status creation time to generate filename 195 | createdAt, err := time.Parse(time.RFC3339, status.CreatedAt.Format(time.RFC3339)) 196 | if err != nil { 197 | log.Printf("Failed to parse created time for status %s: %v", status.ID, err) 198 | continue 199 | } 200 | 201 | // Generate the same filename that would be created 202 | filename := fmt.Sprintf("%s-%s.md", createdAt.Format("2006-01-02"), status.ID) 203 | filePath := filepath.Join(config.ContentDir, filename) 204 | 205 | // Check if file already exists 206 | if _, err := os.Stat(filePath); err == nil { 207 | fmt.Printf(" already exists, skipping\n") 208 | skippedCount++ 209 | continue 210 | } 211 | } 212 | 213 | if err := processStatus(status, config); err != nil { 214 | log.Printf("Failed to process status %s: %v", status.ID, err) 215 | continue 216 | } 217 | 218 | fmt.Printf(" done\n") 219 | processedCount++ 220 | } 221 | 222 | fmt.Printf("Completed! Processed: %d, Skipped: %d\n", processedCount, skippedCount) 223 | 224 | fmt.Println("Done!") 225 | } 226 | 227 | func processStatus(status *mastodon.Status, config Config) error { 228 | // Skip reblogs/boosts - we want original content only 229 | if status.Reblog != nil { 230 | return nil 231 | } 232 | 233 | // Parse the status creation time 234 | createdAt, err := time.Parse(time.RFC3339, status.CreatedAt.Format(time.RFC3339)) 235 | if err != nil { 236 | return fmt.Errorf("failed to parse created time: %v", err) 237 | } 238 | 239 | // Download media attachments 240 | var mediaFiles []string 241 | for _, attachment := range status.MediaAttachments { 242 | if attachment.URL == "" { 243 | continue 244 | } 245 | 246 | filename, err := downloadMedia(attachment.URL, config.MediaDir) 247 | if err != nil { 248 | log.Printf("Failed to download media %s: %v", attachment.URL, err) 249 | continue 250 | } 251 | mediaFiles = append(mediaFiles, filename) 252 | } 253 | 254 | // Create post metadata 255 | metadata := PostMetadata{ 256 | Date: createdAt, 257 | OriginalURL: status.URL, 258 | Media: mediaFiles, 259 | } 260 | 261 | // Generate filename: YYYY-MM-DD-post-id.md 262 | filename := fmt.Sprintf("%s-%s.md", 263 | createdAt.Format("2006-01-02"), 264 | status.ID) 265 | 266 | filepath := filepath.Join(config.ContentDir, filename) 267 | 268 | // Create markdown file 269 | content := stripHTML(status.Content) 270 | 271 | // Remove trailing tag from content if it exists 272 | if config.TrailingTag != "" { 273 | content = removeTrailingTag(content, config.TrailingTag) 274 | } 275 | 276 | return createMarkdownFile(filepath, metadata, content) 277 | } 278 | 279 | func downloadMedia(mediaURL, mediaDir string) (string, error) { 280 | // Parse URL to get filename 281 | parsedURL, err := url.Parse(mediaURL) 282 | if err != nil { 283 | return "", err 284 | } 285 | 286 | // Extract filename from URL path 287 | filename := path.Base(parsedURL.Path) 288 | if filename == "." || filename == "/" { 289 | // Generate filename if not available 290 | filename = fmt.Sprintf("media-%d", time.Now().UnixNano()) 291 | } 292 | 293 | // Create full file path 294 | filePath := filepath.Join(mediaDir, filename) 295 | 296 | // Check if file already exists 297 | if _, err := os.Stat(filePath); err == nil { 298 | return filename, nil // File already exists 299 | } 300 | 301 | // Download the file 302 | resp, err := http.Get(mediaURL) 303 | if err != nil { 304 | return "", err 305 | } 306 | defer resp.Body.Close() 307 | 308 | if resp.StatusCode != http.StatusOK { 309 | return "", fmt.Errorf("failed to download media: HTTP %d", resp.StatusCode) 310 | } 311 | 312 | // Create the file 313 | file, err := os.Create(filePath) 314 | if err != nil { 315 | return "", err 316 | } 317 | defer file.Close() 318 | 319 | // Copy data 320 | _, err = io.Copy(file, resp.Body) 321 | if err != nil { 322 | return "", err 323 | } 324 | 325 | return filename, nil 326 | } 327 | 328 | func createMarkdownFile(filepath string, metadata PostMetadata, content string) error { 329 | file, err := os.Create(filepath) 330 | if err != nil { 331 | return err 332 | } 333 | defer file.Close() 334 | 335 | // Write YAML front matter 336 | _, err = file.WriteString("---\n") 337 | if err != nil { 338 | return err 339 | } 340 | 341 | encoder := yaml.NewEncoder(file) 342 | encoder.SetIndent(2) 343 | if err := encoder.Encode(metadata); err != nil { 344 | return err 345 | } 346 | encoder.Close() 347 | 348 | _, err = file.WriteString("---\n\n") 349 | if err != nil { 350 | return err 351 | } 352 | 353 | // Write content as markdown 354 | _, err = file.WriteString(content) 355 | return err 356 | } 357 | 358 | // stripHTML removes HTML tags from content while preserving line breaks and decoding HTML entities 359 | func stripHTML(content string) string { 360 | // Step 1: Convert HTML line breaks to newlines 361 | content = regexp.MustCompile(`