├── README
├── testdata_num.txt
├── testdata_urls.txt
├── cliargs.php
├── parallelcurl.php
└── buzzprofilecrawl.php


/README:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/petewarden/buzzprofilecrawl/HEAD/README


--------------------------------------------------------------------------------
/testdata_num.txt:
--------------------------------------------------------------------------------
 1 | 114482883275883376711
 2 | 110180527922237151490
 3 | 100722937543579674532
 4 | 111343326546693401371
 5 | 100023042519328780763
 6 | 107298427506852802199
 7 | 115624021056418704315
 8 | 109213761823894503198
 9 | 117827376991976059856
10 | 112036812458594362599
11 | 


--------------------------------------------------------------------------------
/testdata_urls.txt:
--------------------------------------------------------------------------------
 1 | http://www.google.com/profiles/115863474911002159675
 2 | http://www.google.com/profiles/102563990701473154621
 3 | http://www.google.com/profiles/100448244799026528799
 4 | http://www.google.com/profiles/100503905368015720339
 5 | http://www.google.com/profiles/117913250525659179200
 6 | http://www.google.com/profiles/103228038539082456359
 7 | http://www.google.com/profiles/104251208055103372299
 8 | http://www.google.com/profiles/100359947051490088968
 9 | http://www.google.com/profiles/104287775546672278556
10 | http://www.google.com/profiles/113509451750336198219
11 | http://www.google.com/profiles/104945601995852006424
12 | http://www.google.com/profiles/106664725926862859359


--------------------------------------------------------------------------------
/cliargs.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | // A set of utility functions to make handling command line arguments in PHP easier
  4 | // To use them, pass in an array describing the expected arguments, in the form
  5 | //
  6 | // array(
  7 | // '<long name of argument>' => array(
  8 | //     'short' => '<single letter version of argument>',
  9 | //     'type' => <'switch' | 'optional' | 'required'>,
 10 | //     'description' => '<help text for the argument>',
 11 | //     'default' => '<value if this is an optional argument and it isn't specified>',
 12 | // ),
 13 | // ...
 14 | // );
 15 | //
 16 | // If the type is switch, then the result is a boolean that will be false if it's
 17 | // not present, or true if it is
 18 | //
 19 | // If the type is optional, then the result will be the default if it's not present
 20 | //
 21 | // If the type is required, then the script will print out the usage and exit if it's
 22 | // not found
 23 | //
 24 | // To use, call cliargs_print_usage_and_exit() with the array of argument descriptions
 25 | // The result will be an array with the argument names as keys to the found values
 26 | //
 27 | // by Pete Warden, http://petewarden.typepad.com but freely reusable with no restrictions
 28 | 
 29 | function cliargs_print_usage_and_exit($cliargs)
 30 | {
 31 | 	print "Usage:\n";
 32 | 	
 33 | 	foreach ($cliargs as $long => $arginfo)
 34 | 	{
 35 | 		$short = $arginfo['short'];
 36 | 		$type = $arginfo['type'];
 37 | 		$required = ($type=='required');
 38 | 		$optional = ($type=='optional');
 39 | 		$description = $arginfo['description'];
 40 | 		
 41 | 		print "-$short/--$long ";
 42 | 
 43 | 		if ($optional||$required)
 44 | 			print "<value> ";
 45 | 
 46 | 		print ": $description";
 47 | 
 48 | 		if ($required)
 49 | 			print " (required)";
 50 | 
 51 | 		print "\n";
 52 | 	}
 53 | 	
 54 | 	exit();
 55 | }
 56 | 
 57 | function cliargs_strstartswith($source, $prefix)
 58 | {
 59 |    return strncmp($source, $prefix, strlen($prefix)) == 0;
 60 | }
 61 | 
 62 | function cliargs_get_options($cliargs)
 63 | {
 64 | 	global $argv;
 65 | 	global $argc;
 66 | 
 67 | 	$options = array('unnamed' => array());
 68 | 	for ($index=1; $index<$argc; $index+=1)
 69 | 	{
 70 | 		$currentarg = strtolower($argv[$index]);
 71 | 		$argparts = split('=', $currentarg);
 72 | 		$namepart = $argparts[0];
 73 | 		
 74 | 		if (cliargs_strstartswith($namepart, '--'))
 75 | 		{
 76 | 			$longname = substr($namepart, 2);
 77 | 		}
 78 | 		else if (cliargs_strstartswith($namepart, '-'))
 79 | 		{
 80 | 			$shortname = substr($namepart, 1);
 81 |             $longname = $shortname;
 82 | 			foreach ($cliargs as $name => $info)
 83 | 			{
 84 | 				if ($shortname===$info['short'])
 85 | 				{
 86 | 					$longname = $name;
 87 | 					break;
 88 | 				}
 89 | 			}
 90 | 		
 91 | 		}
 92 | 		else
 93 | 		{
 94 | 			$longname = 'unnamed';
 95 | 		}
 96 | 		
 97 | 		if ($longname=='unnamed')
 98 | 		{				
 99 | 			$options['unnamed'][] = $namepart;
100 | 		}
101 | 		else
102 | 		{
103 | 			if (empty($cliargs[$longname]))
104 | 			{
105 | 				print "Unknown argument '$longname'\n";
106 | 				cliargs_print_usage_and_exit($cliargs);
107 | 			}
108 | 			
109 | 			$arginfo = $cliargs[$longname];
110 | 			$argtype = $arginfo['type'];
111 | 			if ($argtype==='switch')
112 | 			{
113 | 				$value = true;
114 | 			}
115 | 			else if (isset($argparts[1]))
116 | 			{
117 | 				$value = $argparts[1];
118 | 			}
119 | 			else if (($index+1)<$argc)
120 | 			{
121 | 				$value = $argv[$index+1];
122 | 				$index += 1;
123 | 			}
124 | 			else
125 | 			{
126 | 				print "Missing value after '$longname'\n";
127 | 				cliargs_print_usage_and_exit($cliargs);
128 | 			}
129 | 			
130 | 			$options[$longname] = $value;
131 | 		}
132 | 	}
133 | 
134 | 	foreach ($cliargs as $longname => $arginfo)
135 | 	{
136 | 		$type = $arginfo['type'];
137 | 
138 | 		if (!isset($options[$longname]))
139 | 		{
140 | 			if ($type=='required')
141 | 			{
142 | 				print("Missing required value for '$longname'\n");
143 | 				cliargs_print_usage_and_exit($cliargs);
144 | 			}
145 | 			else if ($type=='optional')
146 | 			{
147 | 				if (!isset($arginfo['default']))
148 | 					die('Missing default value for '.$long);
149 | 					
150 | 				$options[$longname] = $arginfo['default'];
151 | 			}
152 | 			else if ($type=='switch')
153 | 			{
154 | 				$options[$longname] = false;
155 | 			}
156 | 		}
157 | 	}
158 | 	
159 | 	return $options;
160 | }
161 | 
162 | ?>


--------------------------------------------------------------------------------
/parallelcurl.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | // This class is designed to make it easy to run multiple curl requests in parallel, rather than
  4 | // waiting for each one to finish before starting the next. Under the hood it uses curl_multi_exec
  5 | // but since I find that interface painfully confusing, I wanted one that corresponded to the tasks
  6 | // that I wanted to run.
  7 | //
  8 | // To use it, first create the ParallelCurl object:
  9 | //
 10 | // $parallelcurl = new ParallelCurl(10);
 11 | //
 12 | // The first argument to the constructor is the maximum number of outstanding fetches to allow
 13 | // before blocking to wait for one to finish. You can change this later using setMaxRequests()
 14 | // The second optional argument is an array of curl options in the format used by curl_setopt_array()
 15 | //
 16 | // Next, start a URL fetch:
 17 | //
 18 | // $parallelcurl->startRequest('http://example.com', 'on_request_done', array('something'));
 19 | //
 20 | // The first argument is the address that should be fetched
 21 | // The second is the callback function that will be run once the request is done
 22 | // The third is a 'cookie', that can contain arbitrary data to be passed to the callback
 23 | //
 24 | // This startRequest call will return immediately, as long as less than the maximum number of
 25 | // requests are outstanding. Once the request is done, the callback function will be called, eg:
 26 | //
 27 | // on_request_done($content, 'http://example.com', $ch, array('something));
 28 | //
 29 | // The callback should take four arguments. The first is a string containing the content found at
 30 | // the URL. The second is the original URL requested, the third is the curl handle of the request that
 31 | // can be queried to get the results, and the fourth is the arbitrary 'cookie' value that you 
 32 | // associated with this object. This cookie contains user-defined data.
 33 | //
 34 | // Since you may have requests outstanding at the end of your script, you *MUST* call
 35 | //
 36 | // $parallelcurl->finishAllRequests();
 37 | //
 38 | // before you exit. If you don't, the final requests may be left unprocessed!
 39 | //
 40 | // By Pete Warden <pete@petewarden.com>, freely reusable, see http://petewarden.typepad.com for more
 41 | 
 42 | class ParallelCurl {
 43 | 
 44 |     public $max_requests;
 45 |     public $options;
 46 | 
 47 |     public $outstanding_requests;
 48 |     public $multi_handle;
 49 |     
 50 |     public function __construct($in_max_requests = 10, $in_options = array()) {
 51 |         $this->max_requests = $in_max_requests;
 52 |         $this->options = $in_options;
 53 |         
 54 |         $this->outstanding_requests = array();
 55 |         $this->multi_handle = curl_multi_init();
 56 |     }
 57 | 
 58 |     // Sets how many requests can be outstanding at once before we block and wait for one to
 59 |     // finish before starting the next one
 60 |     public function setMaxRequests($in_max_requests) {
 61 |         $max_requests = $in_max_requests;
 62 |     }
 63 |     
 64 |     // Sets the options to pass to curl, using the format of curl_setopt_array()
 65 |     public function setOptions($in_options) {
 66 | 
 67 |         $options = $in_options;
 68 |     }
 69 | 
 70 |     // Start a fetch from the $url address, calling the $callback function passing the optional
 71 |     // $user_data value. The callback should accept 3 arguments, the url, curl handle and user
 72 |     // data, eg on_request_done($url, $ch, $user_data);
 73 |     public function startRequest($url, $callback, $user_data = array()) {
 74 | 
 75 |         $this->waitForOutstandingRequestsToDropBelow($this->max_requests);
 76 |     
 77 |         $ch = curl_init();
 78 |         curl_setopt_array($ch, $this->options);
 79 |         curl_setopt($ch, CURLOPT_URL, $url);
 80 |         curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
 81 |         
 82 |         curl_multi_add_handle($this->multi_handle, $ch);
 83 |         
 84 |         $this->outstanding_requests[$ch] = array(
 85 |             'url' => $url,
 86 |             'callback' => $callback,
 87 |             'user_data' => $user_data,
 88 |         );
 89 |         
 90 |         $this->checkForCompletedRequests();
 91 |     }
 92 |     
 93 |     // You *MUST* call this function at the end of your script. It waits for any running requests
 94 |     // to complete, and calls their callback functions
 95 |     public function finishAllRequests() {
 96 |         $this->waitForOutstandingRequestsToDropBelow(1);
 97 |     }
 98 | 
 99 |     // Checks to see if any of the outstanding requests have finished
100 |     private function checkForCompletedRequests() {
101 | 
102 |         // Call select to see if anything is waiting for us
103 |         if (curl_multi_select($this->multi_handle, 0.0) === -1)
104 |             return;
105 |         
106 |         // Since something's waiting, give curl a chance to process it
107 |         do {
108 |             $mrc = curl_multi_exec($this->multi_handle, $active);
109 |         } while ($mrc == CURLM_CALL_MULTI_PERFORM);
110 |         
111 |         // Now grab the information about the completed requests
112 |         while ($info = curl_multi_info_read($this->multi_handle)) {
113 |         
114 |             $ch = $info['handle'];
115 |             
116 |             if (!isset($this->outstanding_requests[$ch])) {
117 |                 die("Error - handle wasn't found in requests: '$ch' in ".
118 |                     print_r($this->outstanding_requests, true));
119 |             }
120 |             
121 |             $request = $this->outstanding_requests[$ch];
122 | 
123 |             $url = $request['url'];
124 |             $content = curl_multi_getcontent($ch);
125 |             $callback = $request['callback'];
126 |             $user_data = $request['user_data'];
127 |             
128 |             $callback($content, $url, $ch, $user_data);
129 |             
130 |             unset($this->outstanding_requests[$ch]);
131 |             
132 |             curl_multi_remove_handle($this->multi_handle, $ch);
133 |         }
134 |     
135 |     }
136 |     
137 |     // Blocks until there's less than the specified number of requests outstanding
138 |     private function waitForOutstandingRequestsToDropBelow($max)
139 |     {
140 |         while (count($this->outstanding_requests)>=$max)
141 |         {
142 |             $this->checkForCompletedRequests();
143 |             sleep(1);
144 |         }
145 |     }
146 | 
147 | }
148 | 
149 | 
150 | ?>


--------------------------------------------------------------------------------
/buzzprofilecrawl.php:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/php
  2 | <?php
  3 | 
  4 | /**
  5 | *************WARNING*****************
  6 | By running this script you're using resources on Google's servers, so be respectful of their 
  7 | commitment to an open and crawlable web. Check http://google.com/robots.txt at least once a week to
  8 | ensure you're abiding by their site rules, don't fire too many requests at once, and make sure you
  9 | supply a valid contact email so they can get in touch in case of problems.
 10 | *************************************
 11 | 
 12 | This script allows you to download public Google Profiles and extract structured information from
 13 | the HTML. 
 14 | 
 15 | To test it, go to the command line, cd to this folder and run
 16 | 
 17 | ./buzzprofilecrawl.php -f "testdata_*.txt" -e <email> -o <organization>
 18 | 
 19 | where <email> and <organization> are replaced by your contact email address and company, so that
 20 | Google can get in touch if your crawling causes any problems. You should see JSON arrays of
 21 | information for each of the 20 users mentioned in the test data files. 
 22 | 
 23 | The script fetches the HTML for the page from Google's servers, and then runs a set of regular
 24 | expressions to extract the microformatted information for that user. The profiles mostly use hcard
 25 | to help robots like us understand what the meaning of the different elements is.
 26 | 
 27 | The output is in the form <userid><tab character><json data>, eg:
 28 | 
 29 | 106664725926862859359	{"user_name":"searchbrowser","name":"Pete Warden","portrait_url":"\/s2\/photos\/public\/AIbEiAIAAABDCN_Y_J-1nfe-XCILdmNhcmRfcGhvdG8qKDdkYTYyODgxMTAzYjg0OGUzODAzNjM1OTUxMzgxMWVhNjY3MzdlZDgwAUQ6MaRMKXz3oZLOOF-uOVBoUoqx","location":"Boulder, CO","location_born":"Cambridge, UK","employment_history":["Apple"],"education_history":["University of Manchester"],"links":["http:\/\/petewarden.typepad.com\/"],"title":"Software Engineer","organization":"Mailana Inc","location_history":["Dundee, Scotland","Los Angeles, CA"]}
 30 |  
 31 | (c) Pete Warden <pete@petewarden.com> http://petewarden.typepad.com/ Jan 8th 2010
 32 | 
 33 | Redistribution and use in source and binary forms, with or without modification, are
 34 | permitted provided that the following conditions are met:
 35 | 
 36 | 1. Redistributions of source code must retain the above copyright notice, this 
 37 |   list of conditions and the following disclaimer.
 38 | 2. Redistributions in binary form must reproduce the above copyright notice, this 
 39 |   list of conditions and the following disclaimer in the documentation and/or 
 40 |   other materials provided with the distribution.
 41 | 3. The name of the author may not be used to endorse or promote products derived 
 42 |   from this software without specific prior written permission.
 43 | 
 44 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
 45 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 46 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 47 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
 48 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
 49 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
 50 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 51 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
 52 | OF SUCH DAMAGE.
 53 | 
 54 |  */
 55 | 
 56 | require_once('cliargs.php');
 57 | require_once('parallelcurl.php');
 58 | 
 59 | define('SOURCE_USER_ID_RE', '@http://www.google.com/profiles/([0-9]+)@');
 60 | define('SOURCE_USER_NAME_RE', '@http://www.google.com/profiles/([^/]+)@');
 61 | 
 62 | // These are the REs used to extract the information from the raw HTML. Most of the
 63 | // elements are defined by the hCard microformat, for more details see
 64 | // http://microformats.org/wiki/hcard
 65 | define('NAME_RE', '@<span class="fn">([^<]*)</span>@');
 66 | define('PORTRAIT_RE', '@<img class="ll_profilephoto photo" src="([^"]+)"@');
 67 | define('LOCATION_RE', '@<span class="adr">([^<]*)</span>@');
 68 | define('TITLE_RE', '@<div class="tagline"><p>([^<]*) at@');
 69 | define('SCHOOL_RE', '@<span class="school">([^<]*)</span>@');
 70 | define('LOCATION_BORN_RE', '@<dt>Where I grew up</dt><dd>([^<]*)</dd>@');
 71 | define('WORK_HISTORY_RE', '@<dt>Companies I&#39;ve worked for</dt><dd>([^<]*)</dd>@');
 72 | define('EDUCATION_HISTORY_RE', '@<dt>Schools I&#39;ve attended</dt><dd>([^<]*)</dd>@');
 73 | define('OTHER_NAME_RE', '@<dt>Other names</dt><dd>([^<]*)</dd></dl></div>@');
 74 | define('LINKS_RE', '@<div class="link"><a href="([^"]+)">@');
 75 | define('TITLE_ALT_RE', '@<span class="title">([^<]*)</span>@');
 76 | define('ORGANIZATION_RE', '@<span class="org">([^<]*)</span>@');
 77 | define('LOCATION_HISTORY_RE', '@<dt>Places I&#39;ve lived</dt><dd>([^<]*)</dd>@');
 78 | define('MENTIONS_RE', '@/profiles/([0-9]{21,21})@');
 79 | 
 80 | $contentrelist = array(
 81 |     NAME_RE => array('name' => 'name'),
 82 |     PORTRAIT_RE => array('name' => 'portrait_url'),
 83 |     LOCATION_RE => array('name' => 'location'),
 84 |     TITLE_RE => array('name' => 'title'),
 85 |     SCHOOL_RE => array('name' => 'organization'),
 86 |     LOCATION_BORN_RE => array('name' => 'location_born'),
 87 |     WORK_HISTORY_RE => array('name' => 'employment_history', 'list' => true),
 88 |     EDUCATION_HISTORY_RE => array('name' => 'education_history', 'list' => true),
 89 |     OTHER_NAME_RE => array('name' => 'other_names', 'list' => true),
 90 |     LINKS_RE => array('name' => 'links', 'multiple' => true),
 91 |     TITLE_ALT_RE => array('name' => 'title'),
 92 |     ORGANIZATION_RE => array('name' => 'organization'),
 93 |     LOCATION_HISTORY_RE => array('name' => 'location_history', 'list' => true),
 94 |     MENTIONS_RE => array('name' => 'mentions', 'multiple' => true),
 95 | );
 96 | 
 97 | define('FETCH_TIMEOUT', 30);
 98 | 
 99 | // This function is called when ParallelCurl completes a page fetch, and it handles converting
100 | // the HTML into structured JSON data that's printed to stdout.
101 | function parse_page($content, $url, $ch, $userdata)
102 | {
103 | 	global $contentrelist;
104 | 
105 |     $redirecturl = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
106 | 
107 | 	if (empty($content))
108 | 		return null;
109 | 
110 |     $content = str_replace("\n", "", $content);
111 | 
112 |     if (!preg_match(SOURCE_USER_ID_RE, $url, $matches))
113 |         return null;
114 | 
115 |     $userid = $matches[1];
116 | 
117 |     if (!preg_match(SOURCE_USER_NAME_RE, $redirecturl, $matches))
118 |         return null;
119 | 
120 |     $username = $matches[1];
121 |     if ($username==$userid)
122 |         $username = '';
123 |     
124 |     $result = array();
125 |     if (!empty($username))
126 |         $result['user_name'] = $username;
127 |     
128 | 	foreach ($contentrelist as $currentre => $reinfo)
129 | 	{
130 | 		if (!preg_match_all($currentre, $content, $matches))
131 |             continue;
132 | 
133 |         if (isset($reinfo['multiple']))
134 |         {
135 |             $matcharray = $matches[1];
136 |             $output = array();
137 |             foreach ($matcharray as $matchtext)
138 |                 $output[] = htmlspecialchars_decode($matchtext, ENT_QUOTES);
139 |         }
140 |         else if (isset($reinfo['list']))
141 |         {
142 |             $matchtext = htmlspecialchars_decode($matches[1][0], ENT_QUOTES);
143 |             $output = explode('; ', $matchtext);
144 |         }
145 |         else
146 |         {
147 |             $matchtext = htmlspecialchars_decode($matches[1][0], ENT_QUOTES);
148 |             $output = $matchtext;;
149 |         }
150 |         $name = $reinfo['name'];
151 |         
152 |         if (!empty($output))
153 |             $result[$name] = $output;
154 | 	}
155 |     
156 |     print $userid."\t".json_encode($result)."\n";
157 | }
158 | 
159 | $cliargs = array(
160 | 	'filepattern' => array(
161 | 		'short' => 'f',
162 | 		'type' => 'required',
163 | 		'description' => 'The files to read the URLs from',
164 | 	),
165 |     'organization' => array(
166 |         'short' => 'o',
167 |         'type' => 'required',
168 |         'description' => 'The name of the organization or company running this crawler',
169 |     ),
170 |     'email' => array(
171 |         'short' => 'e',
172 |         'type' => 'required',
173 |         'description' => 'An email address where server owners can report any problems with this crawler',
174 |     ),
175 |     'threads' => array(
176 |         'short' => 't',
177 |         'type' => 'optional',
178 |         'description' => 'How many to requests to run at the same time',
179 |         'default' => 1,
180 |     ),
181 | );
182 | 
183 | $options = cliargs_get_options($cliargs);
184 | $filepattern = $options['filepattern'];
185 | $organization = $options['organization'];
186 | $email = $options['email'];
187 | $threads = $options['threads'];
188 | 
189 | if (empty($organization) || empty($email) || (!strpos($email, '@')))
190 |     die("You need to specify a valid organization and email address\n");
191 | 
192 | $agent = 'Crawler from '.$organization;
193 | $agent .= ' - contact '.$email;
194 | $agent .= ' to report any problems with my crawling. Based on code from http://petewarden.typepad.com';
195 | 
196 | $curloptions = array(
197 |     CURLOPT_SSL_VERIFYPEER => FALSE,
198 |     CURLOPT_SSL_VERIFYHOST => FALSE,
199 | 	CURLOPT_FOLLOWLOCATION => TRUE,
200 | 	CURLOPT_USERAGENT => $agent,
201 | 	CURLOPT_TIMEOUT => FETCH_TIMEOUT,
202 | );
203 | 
204 | $parallelcurl = new ParallelCurl($threads, $curloptions);
205 | 
206 | // Loop through all the files, extract all the URLs and process them
207 | foreach (glob($filepattern) as $filename) 
208 | {
209 |     error_log("Reading $filename");
210 |     
211 |     $filehandle = fopen($filename, 'r');
212 |     
213 |     $usertotal = 0;
214 |     
215 |     while(!feof($filehandle))
216 |     {
217 |         $currentline = fgets($filehandle);
218 |         $currenturl = trim($currentline);
219 | 
220 |         $usertotal += 1;
221 |         if (($usertotal%10000)===0)
222 |             error_log(number_format($usertotal).' users processed');
223 | 
224 |         if (empty($currenturl))
225 |             continue;
226 | 
227 |         if (is_numeric($currenturl))
228 |             $currenturl = 'http://www.google.com/profiles/'.$currenturl;
229 |     
230 |         $parallelcurl->startRequest($currenturl, 'parse_page');
231 |     }
232 | 
233 |     fclose($filehandle);
234 | }
235 | 
236 | // Important - if you remove this any pending requests may not be processed
237 | $parallelcurl->finishAllRequests();
238 | 
239 | ?>


--------------------------------------------------------------------------------