├── insta.csv
├── fb.csv
├── twitter.csv
├── you.csv
├── crawlyoutube.sh
├── crawlinsta.sh
├── README.md
├── crawltwitter.sh
├── crawlfb.sh
└── LICENSE
/insta.csv:
--------------------------------------------------------------------------------
1 | Aurel,https://www.instagram.com/aurelie.hermansyah/
2 |
--------------------------------------------------------------------------------
/fb.csv:
--------------------------------------------------------------------------------
1 | ITS,https://www.facebook.com/InstitutTeknologiSepuluhNopember/
2 |
--------------------------------------------------------------------------------
/twitter.csv:
--------------------------------------------------------------------------------
1 | Aurel,https://twitter.com/aurelhermansyah
2 | Ashanti,https://twitter.com/ashantysiddik
3 |
--------------------------------------------------------------------------------
/you.csv:
--------------------------------------------------------------------------------
1 | okezone,https://www.youtube.com/user/okezoneindonesia/videos
2 | RCTI,https://www.youtube.com/user/RCTIOfficialChannel/videos
3 |
--------------------------------------------------------------------------------
/crawlyoutube.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -l
2 | IFS=","
3 | while
4 | read f1 f2
5 | do
6 | nama=`echo $f1 | tr ' ' '-' | tr '\t' '-'`
7 | echo "id $nama"
8 | url=${f2%?}
9 | if [ "$url" != "-" ]
10 | then
11 | status=$(curl -A "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8)" -w "%{http_code}" -o temp -L --silent "$url")
12 | if [[ "$status" =~ "200" ]]
13 | then
14 | now=$(date +"%d-%m-%y-%H-%M")
15 | grep "yt-lockup-title" temp | grep -Eoh 'title=".*" a'| sed -r 's/^.{7}//' | sed -r 's/.{4}$//' > feed
16 | grep "yt-lockup-meta-info" temp | grep -Eoh ".*" | sed -r 's/^.{9}//' | sed -r 's/.{10}$//' > tgl
17 | paste tgl feed | tr '[:upper:]' '[:lower:]'> res/$nama-$now-y.tsv
18 | fi
19 | fi
20 | done < $1
21 |
22 |
23 |
--------------------------------------------------------------------------------
/crawlinsta.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -l
2 | IFS=","
3 | while
4 | read f1 f2
5 | do
6 | nama=`echo $f1 | tr ' ' '-' | tr '\t' '-'`
7 | echo "id $nama"
8 | url=${f2%?}
9 | if [ "$url" != "-" ]
10 | then
11 | status=$(curl -A "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8)" -w "%{http_code}" -o temp -L --silent "$url")
12 | if [[ "$status" =~ "200" ]]
13 | then
14 | now=$(date +"%d-%m-%y-%H-%M")
15 | grep 'window._sharedData' temp | sed -r 's/^.{52}//' | sed -r 's/.{11}$//' > temp1
16 | grep -Po '"date":.*?[^\\]",' temp1 | grep "caption" > temp
17 | grep -Eoh '"date":[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' temp | sed -r 's/^.{7}//' > tgl
18 | grep -Po '"caption":.*?[^\\]",' temp | sed -r 's/^.{11}//' | sed -r 's/.{2}$//'> feed
19 | > tgl1
20 | while
21 | read line
22 | do
23 | date -d @$line +"%d-%m-%y %T" >> tgl1
24 | done < tgl
25 | paste tgl1 feed | tr '[:upper:]' '[:lower:]'> res/$nama-$now-i.tsv
26 | rm temp temp1 feed tgl1 tgl
27 | fi
28 | fi
29 | done < $1
30 |
31 |
32 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # crawlsocmed
2 | simple scripts for crawling data from a social media without its API and username
3 |
4 | ## Twitter
5 | 1. create a directory and name it "res"
6 | 2. create a file (e.g: twitter.csv) containing list of ids and the URLs of the Twitter account
7 | 3. run the script ./crawltwitter.sh twitter.csv
8 | 4. look at your results in the res directory
9 |
10 | ## Facebook
11 | 1. create a directory and name it "res"
12 | 2. create a file (e.g: fb.csv) containing list of ids and the URLs of the Facebook page
13 | 3. run the script ./crawlfb.sh fb.csv
14 | 4. look at your results in the res directory
15 |
16 |
17 | ## Instagram
18 | 1. create a directory and name it "res"
19 | 2. create a file (e.g: insta.csv) containing list of ids and the URLs of the Instagram account
20 | 3. run the script ./crawlinsta.sh insta.csv
21 | 4. look at your results in the res directory
22 |
23 |
24 | ## Youtube
25 | 1. create a directory and name it "res"
26 | 2. create a file (e.g: you.csv) containing list of ids and the URLs of the youtube channel/videos
27 | 3. run the script ./crawlyoutube.sh you.csv
28 | 4. look at your results in the res directory
29 |
--------------------------------------------------------------------------------
/crawltwitter.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -l
2 | IFS=","
3 | while
4 | read f1 f2
5 | do
6 | nama=`echo $f1 | tr ' ' '-' | tr '\t' '-'`
7 | echo "id $nama"
8 | url=${f2%?}
9 | if [ "$url" != "-" ]
10 | then
11 | status=$(curl -A "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8)" -w "%{http_code}" -o temp -L --silent "$url")
12 | if [[ "$status" =~ "200" ]]
13 | then
14 | now=$(date +"%d-%m-%y-%H-%M")
15 | grep -E "(TweetTextSize)|(js-retweet-text)" temp | sed -n '/^$/!{s/<[^>]*>//g;p;}' | tr '[:upper:]' '[:lower:]' > feed
16 | grep -oh "[0-9][0-9]\.[0-9][0-9] - [0-9][0-9]* [A-Z][a-z][a-z] [0-9][0-9][0-9][0-9]" temp | uniq| sed '1d' > tgl
17 | rt=false
18 | > temp
19 | while
20 | read line
21 | do
22 | if [[ "$line" =~ "retweet" ]]
23 | then
24 | rt=true
25 | continue
26 | fi
27 | if [ "$rt" = true ]
28 | then
29 | echo " rt $line"
30 | echo " rt $line" >> temp
31 | rt=false
32 | else
33 | echo "$line"
34 | echo "$line" >> temp
35 | fi
36 |
37 |
38 | done < feed
39 | paste tgl temp > "res/$nama-$now-t.tsv"
40 | rm tgl feed temp
41 | fi
42 | fi
43 | done < $1
44 |
45 | #