├── BudgetData.json
├── README.md
├── src
    └── simpleads
    │   └── io
    │       └── bittiger
    │           ├── ads
    │               ├── Ad.java
    │               ├── AdsAllocation.java
    │               ├── AdPricing.java
    │               ├── AdsRanker.java
    │               ├── AdsFilter.java
    │               ├── AdsServer.java
    │               ├── AdsCampaignManager.java
    │               ├── AdsSelector.java
    │               ├── QueryParser.java
    │               ├── IndexBuilder.java
    │               └── AdsEngine.java
    │           └── ut
    │               ├── QueryUnderstandUT.java
    │               ├── AdEngineUT.java
    │               ├── AdsRankerUT.java
    │               └── IndexBuilderUT.java
└── AdsData.json


/BudgetData.json:
--------------------------------------------------------------------------------
1 | {
2 |   "campaigns":
3 |   [
4 |     {"campaignId":66,"budget":1500},
5 |     {"campaignId":67,"budget":2800},
6 |     {"campaignId":68,"budget":900}
7 |   ]
8 | }
9 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # miniSearchAds
 2 | build simplified search ads system
 3 | 
 4 | Environment configuration
 5 | 
 6 | Programming language: Java 
 7 | 
 8 | IDE: Eclipse/IntelliJ
 9 | 
10 | Key Value store: Memcached Server(http://tugdualgrall.blogspot.com/2011/11/installing-memcached-on-mac-os-x-and.html)
11 | 
12 | Lib: Lucene-core (NLP), spymemcached (Memcached client), junit(java unit test)
13 | 
14 | 
15 | 
16 | Business logic
17 | 
18 | 1. Query Understand
19 | 2. Select Ads
20 | 3. Filter Ads
21 | 4. Rank Ads
22 | 5. Select Top K Ads
23 | 6. Pricing
24 | 7. Allocate Ads
25 | 
26 | 
27 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/Ad.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.io.Serializable;
 4 | import java.util.List;
 5 | 
 6 | public class Ad implements Serializable{
 7 | 	/**
 8 | 	 * 
 9 | 	 */
10 | 	private static final long serialVersionUID = 1L;
11 | 	public Long adId;
12 | 	public Long campaignId;
13 | 	public List<String> keyWords;
14 | 	public double relevanceScore;
15 | 	public double pClick;	
16 | 	public double bidPrice;
17 | 	public double rankScore;
18 | 	public double qualityScore;
19 | 	public double costPerClick;
20 | 	public int position;//1: main line, 2: side bar
21 | }
22 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ut/QueryUnderstandUT.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ut;
 2 | 
 3 | import static org.junit.Assert.*;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | import org.junit.Test;
 8 | 
 9 | import simpleads.io.bittiger.ads.QueryParser;
10 | 
11 | public class QueryUnderstandUT {
12 | 
13 | 	@Test
14 | 	public void testQueryUnderstand() {
15 | 		String query = "looking for the new nike shoe";
16 | 		List<String> queryTerms = QueryParser.getInstance().QueryUnderstand(query);
17 | 		for(String term : queryTerms)
18 | 		{
19 | 			System.out.println("term="+ term);
20 | 		}
21 | 		assertEquals(queryTerms.size(),4);
22 | 	}
23 | 
24 | }
25 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsAllocation.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.util.List;
 4 | 
 5 | public class AdsAllocation {
 6 | 	private static AdsAllocation instance = null;
 7 | 	private static double mainLinePriceThreshold = 4.5;
 8 | 	protected AdsAllocation()
 9 | 	{
10 | 
11 | 	}
12 | 	public static AdsAllocation getInstance() {
13 | 	      if(instance == null) {
14 | 	         instance = new AdsAllocation();
15 | 	      }
16 | 	      return instance;
17 | 	}
18 | 	public void AllocateAds(List<Ad> ads)
19 | 	{
20 | 		for(Ad ad : ads)
21 | 		{
22 | 			if(ad.costPerClick >= mainLinePriceThreshold)
23 | 			{
24 | 				ad.position = 1;
25 | 			}
26 | 			else
27 | 			{
28 | 				ad.position = 2;
29 | 			}
30 | 		}
31 | 	}
32 | }
33 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdPricing.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.util.List;
 4 | 
 5 | public class AdPricing {
 6 | 	private static AdPricing instance = null;
 7 | 	protected AdPricing()
 8 | 	{
 9 | 
10 | 	}
11 | 	public static AdPricing getInstance() {
12 | 	      if(instance == null) {
13 | 	         instance = new AdPricing();
14 | 	      }
15 | 	      return instance;
16 | 	}
17 | 	public void setCostPerClick(List<Ad> adsCandidates)
18 | 	{
19 | 		for(int i = 0; i < adsCandidates.size();i++)
20 | 		{
21 | 			if(i < adsCandidates.size() - 1)
22 | 			{
23 | 				adsCandidates.get(i).costPerClick = adsCandidates.get(i + 1).rankScore / adsCandidates.get(i).qualityScore + 0.01;
24 | 			}
25 | 			else
26 | 			{
27 | 				adsCandidates.get(i).costPerClick = adsCandidates.get(i).bidPrice;
28 | 			}
29 | 		}
30 | 	}
31 | }
32 | 


--------------------------------------------------------------------------------
/AdsData.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "ads":
 3 |   [
 4 |     {"adId":1231,"campaignId":66,"keyWords":["basketball","kobe","shoe","nike"],"pClick":0.03,"bidPrice":6.0},
 5 |     {"adId":1232,"campaignId":66,"keyWords":["running","shoe","nike"],"pClick":0.03,"bidPrice":4.5},
 6 |     {"adId":1233,"campaignId":66,"keyWords":["soccer","shoe","nike"],"pClick":0.03,"bidPrice":4.0},
 7 |     {"adId":1234,"campaignId":67,"keyWords":["running","shoe","adidas"],"pClick":0.03,"bidPrice":7.5},
 8 |     {"adId":1235,"campaignId":67,"keyWords":["soccer","shoe","adidas"],"pClick":0.03,"bidPrice":3.5},
 9 |     {"adId":1236,"campaignId":67,"keyWords":["basketball","shoe","adidas"],"pClick":0.03,"bidPrice":5.5},
10 |     {"adId":1237,"campaignId":68,"keyWords":["hawaii","hotel"],"pClick":0.03,"bidPrice":15.0},
11 |     {"adId":1238,"campaignId":68,"keyWords":["hawaii","flight"],"pClick":0.03,"bidPrice":25.0},
12 |     {"adId":1239,"campaignId":68,"keyWords":["miami","hotel"],"pClick":0.03,"bidPrice":16.0},
13 |     {"adId":1240,"campaignId":68,"keyWords":["miami","flight"],"pClick":0.03,"bidPrice":22.0}
14 |   ]
15 | }
16 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ut/AdEngineUT.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ut;
 2 | 
 3 | import static org.junit.Assert.*;
 4 | 
 5 | import java.util.List;
 6 | 
 7 | import org.junit.Test;
 8 | 
 9 | import simpleads.io.bittiger.ads.Ad;
10 | import simpleads.io.bittiger.ads.AdsEngine;
11 | 
12 | public class AdEngineUT {
13 | 
14 | 	@Test
15 | 	public void testSelectAds() {
16 | 		String adsDataFilePath = "/Users/jiayangan/code/SimpleAds/AdsData.json";
17 | 		String budgetDataFilePath = "/Users/jiayangan/code/SimpleAds/BudgetData.json";
18 | 		String memcachedServer = "127.0.0.1";
19 | 		int memcachedPortal = 11211;
20 | 		AdsEngine adsEngine = new AdsEngine(adsDataFilePath,budgetDataFilePath,memcachedServer,memcachedPortal);
21 | 		if(adsEngine.init())
22 | 		{
23 | 			List<Ad> adsCandidates = adsEngine.selectAds("kobe basketball shoe");
24 | 			assertEquals(adsCandidates.size() , 6);
25 | 			for(Ad ad : adsCandidates)
26 | 			{
27 | 				System.out.println("selected ad id = " + ad.adId);
28 | 				System.out.println("selected ad relevance score = " + ad.relevanceScore);
29 | 			}
30 | 		}
31 | 	}
32 | 
33 | }
34 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsRanker.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.util.Collections;
 4 | import java.util.Comparator;
 5 | import java.util.List;
 6 | 
 7 | public class AdsRanker {
 8 | 	private static AdsRanker instance = null;
 9 | 	protected AdsRanker()
10 | 	{
11 | 
12 | 	}
13 | 	public static AdsRanker getInstance() {
14 | 	      if(instance == null) {
15 | 	         instance = new AdsRanker();
16 | 	      }
17 | 	      return instance;
18 | 	}
19 | 	public List<Ad> rankAds(List<Ad> adsCandidates)
20 | 	{
21 | 		for(Ad ad : adsCandidates)
22 | 		{
23 | 			ad.qualityScore = 0.75 * ad.pClick  +  0.25 * ad.relevanceScore;
24 | 			ad.rankScore = ad.qualityScore * ad.bidPrice;			
25 | 		}
26 | 		//sort by rank score
27 | 		Collections.sort(adsCandidates, new Comparator<Ad>() {
28 | 	        @Override
29 | 	        public int compare(Ad ad2, Ad ad1)
30 | 	        {
31 | 	        	if (ad1.rankScore < ad2.rankScore)
32 | 	        		return -1;
33 | 	        	else if(ad1.rankScore > ad2.rankScore)
34 | 	        		return 1;
35 | 	        	else
36 | 	        		return 0;
37 | 	        }
38 | 	    });
39 | 		return adsCandidates;
40 | 	}
41 | }
42 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsFilter.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.util.ArrayList;
 4 | import java.util.List;
 5 | 
 6 | public class AdsFilter {
 7 | 	private static AdsFilter instance = null;
 8 | 	private static double pClickThreshold = 0.01;
 9 | 	private static double relevanceScoreThreshold = 0.1;
10 | 	private static int mimNumOfAds = 4;
11 | 	protected AdsFilter()
12 | 	{
13 | 
14 | 	}
15 | 	public static AdsFilter getInstance() {
16 | 	      if(instance == null) {
17 | 	         instance = new AdsFilter();
18 | 	      }
19 | 	      return instance;
20 | 	}
21 | 	public List<Ad> LevelZeroFilterAds(List<Ad> adsCandidates)
22 | 	{
23 | 		if(adsCandidates.size() <= mimNumOfAds)
24 | 			return adsCandidates;
25 | 		
26 | 		List<Ad> unfilteredAds = new ArrayList<Ad>();
27 | 		for(Ad ad : adsCandidates)
28 | 		{
29 | 			if(ad.pClick > pClickThreshold && ad.relevanceScore > relevanceScoreThreshold)
30 | 			{
31 | 				unfilteredAds.add(ad);
32 | 			}
33 | 		}
34 | 		return unfilteredAds;
35 | 	}
36 | 	public List<Ad> LevelOneFilterAds(List<Ad> adsCandidates,int k)
37 | 	{
38 | 		if(adsCandidates.size() <= mimNumOfAds)
39 | 			return adsCandidates;
40 | 		
41 | 		List<Ad> unfilteredAds = new ArrayList<Ad>();
42 | 		for(int i = 0; i < Math.min(k, adsCandidates.size());i++)
43 | 		{
44 | 			unfilteredAds.add(adsCandidates.get(i));
45 | 		}
46 | 		return unfilteredAds;
47 | 	}
48 | }
49 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ut/AdsRankerUT.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ut;
 2 | 
 3 | import static org.junit.Assert.*;
 4 | 
 5 | import java.util.ArrayList;
 6 | import java.util.List;
 7 | 
 8 | import org.junit.Test;
 9 | 
10 | import simpleads.io.bittiger.ads.Ad;
11 | import simpleads.io.bittiger.ads.AdsRanker;
12 | 
13 | public class AdsRankerUT {
14 | 
15 | 	@Test
16 | 	public void testRankAds() {
17 | 		Ad ad1 = new Ad();
18 | 		ad1.adId = 1111L;
19 | 		ad1.bidPrice = 3.5;
20 | 		ad1.pClick = 0.3;
21 | 		ad1.relevanceScore = 0.6;
22 | 		
23 | 		Ad ad2 = new Ad();
24 | 		ad2.adId = 1112L;
25 | 		ad2.bidPrice = 3.8;
26 | 		ad2.pClick = 0.25;
27 | 		ad2.relevanceScore = 0.8;
28 | 		
29 | 		Ad ad3 = new Ad();
30 | 		ad3.adId = 1113L;
31 | 		ad3.bidPrice = 3.5;
32 | 		ad3.pClick = 0.4;
33 | 		ad3.relevanceScore = 0.3;
34 | 		
35 | 		Ad ad4 = new Ad();
36 | 		ad4.adId = 1114L;
37 | 		ad4.bidPrice = 5.5;
38 | 		ad4.pClick = 0.2;
39 | 		ad4.relevanceScore = 0.5;
40 | 		List<Ad> adsCandidates = new ArrayList<Ad>();
41 | 		adsCandidates.add(ad1);
42 | 		adsCandidates.add(ad2);
43 | 		adsCandidates.add(ad3);
44 | 		adsCandidates.add(ad4);
45 | 		List<Ad> rankedAds = AdsRanker.getInstance().rankAds(adsCandidates);
46 | 		for(Ad ad : rankedAds)
47 | 		{
48 | 			System.out.println("ad id :"+ad.adId+",ad rank score:"+ad.rankScore);
49 | 		}
50 | 		assertEquals((int)(rankedAds.get(0).rankScore * 100),76);
51 | 		assertEquals((int)(rankedAds.get(1).rankScore * 100),63);
52 | 		assertEquals((int)(rankedAds.get(2).rankScore * 100),55);
53 | 		assertEquals((int)(rankedAds.get(3).rankScore * 100),42);
54 | 	}
55 | 
56 | }
57 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsServer.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.io.BufferedReader;
 4 | import java.io.IOException;
 5 | import java.io.InputStreamReader;
 6 | import java.util.List;
 7 | 
 8 | /*
 9 | 1. Rank : qualityScore * bid ,qualityScore = pClick * relevanceScore
10 | 2. Filter by rank score, pClick, relevance score
11 | 3. Dedupe ads per campaign
12 | 4. Pricing: (next quality score/current quality score) * next bid price = next rank score / current quality
13 | 5. Allocation: only ads with rankScore > mainlineReservePrice can be allocated on mainline. 
14 | 
15 | reference：
16 | http://searchengineland.com/new-adwords-ad-ranking-formula-what-does-it-mean-174946
17 | http://www.wordstream.com/blog/ws/2013/10/24/adwords-ad-rank-algorithm
18 | */ 
19 | public class AdsServer {
20 | 	public static void main(String[] args) throws IOException {
21 | 		if(args.length < 4)
22 | 		{
23 | 			System.out.println("Usage: AdsServer <adsDataFilePath> <budgetDataFilePath> <memcachedServer> <memcachedPortal>");
24 | 			System.exit(0);
25 | 		}
26 | 		String adsDataFilePath = args[0];
27 | 		String budgetDataFilePath = args[1];
28 | 		String memcachedServer = args[2];
29 | 		int memcachedPortal = Integer.parseInt(args[3]);
30 | 		AdsEngine adsEngine = new AdsEngine(adsDataFilePath,budgetDataFilePath,memcachedServer,memcachedPortal);
31 | 		if(adsEngine.init())
32 | 		{
33 | 			System.out.println("Ready to take quey");
34 | 			try{
35 | 				BufferedReader br = new BufferedReader(new InputStreamReader(System.in));				
36 | 				String query;				
37 | 				while((query=br.readLine())!=null){
38 | 					//System.out.println(query);
39 | 					List<Ad> adsCandidates = adsEngine.selectAds(query);
40 | 					for(Ad ad : adsCandidates)
41 | 					{
42 | 						System.out.println("final selected ad id = " + ad.adId);
43 | 						System.out.println("final selected ad rank score = " + ad.rankScore);
44 | 					}
45 | 				}
46 | 					
47 | 			}catch(IOException io){
48 | 				io.printStackTrace();
49 | 			}	
50 | 		}
51 | 	}
52 | }
53 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsCampaignManager.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.io.IOException;
 4 | import java.net.InetSocketAddress;
 5 | import java.util.ArrayList;
 6 | import java.util.HashSet;
 7 | import java.util.List;
 8 | 
 9 | import net.spy.memcached.MemcachedClient;
10 | 
11 | public class AdsCampaignManager {
12 | 	private static AdsCampaignManager instance = null;
13 | 	private int EXP = 7200;
14 | 	private String mMemcachedServer;
15 | 	private int mMemcachedPortal;
16 | 	private static double minPriceThreshold = 1.5;
17 | 	protected AdsCampaignManager(String memcachedServer,int memcachedPortal)
18 | 	{
19 | 		mMemcachedServer = memcachedServer;
20 | 		mMemcachedPortal = memcachedPortal;	
21 | 	}
22 | 	public static AdsCampaignManager getInstance(String memcachedServer,int memcachedPortal) {
23 | 	      if(instance == null) {
24 | 	         instance = new AdsCampaignManager(memcachedServer,memcachedPortal);
25 | 	      }
26 | 	      return instance;
27 | 	}
28 | 	public  List<Ad> DedupeByCampaignId(List<Ad> adsCandidates)
29 | 	{
30 | 		List<Ad> dedupedAds = new ArrayList<Ad>();
31 | 		HashSet<Long> campaignIdSet = new HashSet<Long>();
32 | 		for(Ad ad : adsCandidates)
33 | 		{
34 | 			if(!campaignIdSet.contains(ad.campaignId))
35 | 			{
36 | 				dedupedAds.add(ad);
37 | 				campaignIdSet.add(ad.campaignId);
38 | 			}
39 | 		}
40 | 		return dedupedAds;
41 | 	}
42 | 	public List<Ad> ApplyBudget(List<Ad> adsCandidates)
43 | 	{
44 | 		List<Ad> ads = new ArrayList<Ad>();
45 | 		try
46 | 		{
47 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress(mMemcachedServer,mMemcachedPortal));
48 | 			for(int i = 0; i < adsCandidates.size()  - 1;i++)
49 | 			{
50 | 				Ad ad = adsCandidates.get(i);
51 | 				Long campaignId = ad.campaignId;
52 | 				double budget = (double)cache.get(campaignId.toString());
53 | 				if(ad.costPerClick <= budget && ad.costPerClick >= minPriceThreshold)
54 | 				{
55 | 					ads.add(ad);
56 | 					budget = budget - ad.costPerClick;
57 | 					cache.set(campaignId.toString(), EXP, budget);
58 | 				}
59 | 			}
60 | 		}catch (IOException e) {
61 | 			// TODO Auto-generated catch block
62 | 			e.printStackTrace();
63 | 		}
64 | 		
65 | 		return ads;
66 | 	}
67 | 	
68 | }
69 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsSelector.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.io.IOException;
 4 | import java.net.InetSocketAddress;
 5 | import java.util.ArrayList;
 6 | import java.util.HashMap;
 7 | import java.util.List;
 8 | import java.util.Set;
 9 | 
10 | import net.spy.memcached.MemcachedClient;
11 | public class AdsSelector {
12 | 	private static AdsSelector instance = null;
13 | 	//private int EXP = 7200;
14 | 	private String mMemcachedServer;
15 | 	private int mMemcachedPortal;
16 | 	protected AdsSelector(String memcachedServer,int memcachedPortal)
17 | 	{
18 | 		mMemcachedServer = memcachedServer;
19 | 		mMemcachedPortal = memcachedPortal;	
20 | 	}
21 | 	public static AdsSelector getInstance(String memcachedServer,int memcachedPortal) {
22 | 	      if(instance == null) {
23 | 	         instance = new AdsSelector(memcachedServer, memcachedPortal);
24 | 	      }
25 | 	      return instance;
26 |     }
27 | 	public List<Ad> selectAds(List<String> queryTerms)
28 | 	{
29 | 		List<Ad> adList = new ArrayList<Ad>();
30 | 		HashMap<Long,Integer> matchedAds = new HashMap<Long,Integer>();
31 | 		try {
32 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress(mMemcachedServer,mMemcachedPortal));
33 | 
34 | 			for(String queryTerm : queryTerms)
35 | 			{
36 | 				System.out.println("queryTerm = " + queryTerm);
37 | 				@SuppressWarnings("unchecked")
38 | 				Set<Long>  adIdList = (Set<Long>)cache.get(queryTerm);
39 | 				if(adIdList.size() > 0)
40 | 				{
41 | 					for(Object adId : adIdList)
42 | 					{
43 | 						Long key = (Long)adId;
44 | 						if(matchedAds.containsKey(key))
45 | 						{
46 | 							int count = matchedAds.get(key) + 1;
47 | 							matchedAds.put(key, count);
48 | 						}
49 | 						else
50 | 						{
51 | 							matchedAds.put(key, 1);
52 | 						}
53 | 					}
54 | 				}				
55 | 			}
56 | 			for(Long adId:matchedAds.keySet())
57 | 			{			
58 | 				Ad  ad = (Ad)cache.get(adId.toString());
59 | 				double relevanceScore = (double) (matchedAds.get(adId) * 1.0 / ad.keyWords.size());
60 | 				ad.relevanceScore = relevanceScore;
61 | 				adList.add(ad);
62 | 			}			
63 | 		} catch (IOException e) {
64 | 			// TODO Auto-generated catch block
65 | 			e.printStackTrace();
66 | 		}
67 | 		
68 | 		return adList;
69 | 	}
70 | }
71 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/QueryParser.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | import java.io.IOException;
 3 | import java.io.StringReader;
 4 | import java.util.ArrayList;
 5 | import java.util.List;
 6 | 
 7 | import org.apache.lucene.analysis.*;
 8 | import org.apache.lucene.analysis.core.StopFilter;
 9 | import org.apache.lucene.analysis.en.EnglishAnalyzer;
10 | //import org.apache.lucene.analysis.en.PorterStemFilter;
11 | import org.apache.lucene.analysis.standard.StandardTokenizer;
12 | import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
13 | import org.apache.lucene.analysis.util.CharArraySet;
14 | import org.apache.lucene.util.AttributeFactory;
15 | 
16 | public class QueryParser {
17 | 	private static QueryParser instance = null;
18 | 	protected QueryParser()
19 | 	{
20 | 		
21 | 		
22 | 	}
23 | 	public static QueryParser getInstance() {
24 | 	      if(instance == null) {
25 | 	         instance = new QueryParser();
26 | 	      }
27 | 	      return instance;
28 |     }
29 | 	//remove stop word, tokenize, stem
30 | 	public List<String> QueryUnderstand(String query)
31 | 	{
32 | 		List<String> tokens = new ArrayList<String>();
33 | 		AttributeFactory factory = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
34 | 		Tokenizer tokenizer = new StandardTokenizer(factory);
35 | 		tokenizer.setReader(new StringReader(query));
36 | 		CharArraySet stopWords = EnglishAnalyzer.getDefaultStopSet();
37 |         TokenStream tokenStream = new StopFilter(tokenizer, stopWords);
38 |         //tokenStream = new PorterStemFilter(tokenStream);
39 |         StringBuilder sb = new StringBuilder();
40 |         CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
41 |         try {
42 |         	tokenStream.reset();
43 | 	        while (tokenStream.incrementToken()) {
44 | 	            String term = charTermAttribute.toString();
45 | 	            
46 | 	            tokens.add(term);
47 | 	            sb.append(term + " ");
48 | 	        }
49 | 	        tokenStream.end();
50 | 	        tokenStream.close();
51 | 
52 | 	        tokenizer.close();  
53 | 		} catch (IOException e) {
54 | 			// TODO Auto-generated catch block
55 | 			e.printStackTrace();
56 | 		}
57 |     	System.out.println("QU="+ sb.toString());
58 | 		return tokens;
59 | 	}
60 | }
61 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/IndexBuilder.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ads;
 2 | 
 3 | import java.io.IOException;
 4 | import java.net.InetSocketAddress;
 5 | import java.util.HashSet;
 6 | import java.util.Set;
 7 | 
 8 | import net.spy.memcached.MemcachedClient;
 9 | 
10 | public class IndexBuilder {
11 | 	private int EXP = 7200;
12 | 	private String mMemcachedServer;
13 | 	private int mMemcachedPortal;
14 | 	public IndexBuilder(String memcachedServer,int memcachedPortal)
15 | 	{
16 | 		mMemcachedServer = memcachedServer;
17 | 		mMemcachedPortal = memcachedPortal;
18 | 	}
19 | 	public Boolean buildInvertIndex(Ad ad)
20 | 	{
21 | 		try 
22 | 		{
23 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress(mMemcachedServer, mMemcachedPortal));		
24 | 			for(int i = 0; i < ad.keyWords.size();i++)
25 | 			{
26 | 				String key = ad.keyWords.get(i);
27 | 				if(cache.get(key) instanceof Set)
28 | 				{
29 | 					@SuppressWarnings("unchecked")
30 | 					Set<Long>  adIdList = (Set<Long>)cache.get(key);
31 | 					adIdList.add(ad.adId);
32 | 				    cache.set(key, EXP, adIdList);
33 | 				}
34 | 				else
35 | 				{
36 | 					Set<Long>  adIdList = new HashSet<Long>();
37 | 					adIdList.add(ad.adId);
38 | 					cache.set(key, EXP, adIdList);
39 | 				}
40 | 			}
41 | 			
42 | 		} catch (IOException e) {
43 | 			// TODO Auto-generated catch block
44 | 			e.printStackTrace();
45 | 			return false;
46 | 		}
47 | 		return true;
48 | 	}
49 | 	public Boolean buildForwardIndex(Ad ad)
50 | 	{
51 | 		try 
52 | 		{
53 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress(mMemcachedServer, mMemcachedPortal));
54 | 			String key = ad.adId.toString();
55 | 			cache.set(key, EXP, ad);
56 | 		} catch (IOException e) {
57 | 			// TODO Auto-generated catch block
58 | 			e.printStackTrace();
59 | 			return false;
60 | 		}		
61 | 		return true;
62 | 	}
63 | 	public Boolean updateBudget(Long campaignId,double budget)
64 | 	{
65 | 		try 
66 | 		{
67 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress(mMemcachedServer, mMemcachedPortal));
68 | 			String key = campaignId.toString();
69 | 			cache.set(key, EXP, budget);
70 | 		} catch (IOException e) {
71 | 			// TODO Auto-generated catch block
72 | 			e.printStackTrace();
73 | 			return false;
74 | 		}		
75 | 		return true;
76 | 	}
77 | }
78 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ut/IndexBuilderUT.java:
--------------------------------------------------------------------------------
 1 | package simpleads.io.bittiger.ut;
 2 | 
 3 | import static org.junit.Assert.*;
 4 | 
 5 | import java.io.IOException;
 6 | import java.net.InetSocketAddress;
 7 | import java.util.ArrayList;
 8 | import java.util.Set;
 9 | 
10 | import net.spy.memcached.MemcachedClient;
11 | 
12 | import org.junit.Test;
13 | 
14 | import simpleads.io.bittiger.ads.Ad;
15 | import simpleads.io.bittiger.ads.IndexBuilder;
16 | 
17 | public class IndexBuilderUT {
18 | 
19 | 	@Test
20 | 	public void testBuildInvertIndex() {
21 | 		IndexBuilder indexBuilder = new IndexBuilder("127.0.0.1",11211);
22 | 		Ad ad1 = new Ad();
23 | 		ad1.adId = 1121L;
24 | 		ad1.keyWords = new ArrayList<String>();
25 | 		ad1.keyWords.add("basketball");
26 | 		ad1.keyWords.add("kobe");
27 | 		ad1.keyWords.add("shoe");
28 | 		ad1.keyWords.add("nike");
29 | 		
30 | 		Ad ad2 = new Ad();
31 | 		ad2.adId = 1122L;
32 | 		ad2.keyWords = new ArrayList<String>();
33 | 		ad2.keyWords.add("basketball");
34 | 		ad2.keyWords.add("shoe");
35 | 		ad2.keyWords.add("adidas");
36 | 		indexBuilder.buildInvertIndex(ad1);
37 | 		indexBuilder.buildInvertIndex(ad2);
38 | 		
39 | 		try {
40 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress("127.0.0.1",11211));
41 | 			@SuppressWarnings("unchecked")
42 | 			Set<Long>  adIdList = (Set<Long>)cache.get("basketball");
43 | 			System.out.println("ad id list size = " + adIdList.size());
44 | 			assertEquals(adIdList.size() , 2);
45 | 			for(Object id : adIdList)
46 | 			{
47 | 				System.out.println("ad id = " + (Long)(id));
48 | 			}
49 | 			
50 | 		} catch (IOException e) {
51 | 			// TODO Auto-generated catch block
52 | 			e.printStackTrace();
53 | 		}
54 | 
55 | 		
56 | 	}
57 | 
58 | 	@Test
59 | 	public void testBuildForwardIndex() {
60 | 		IndexBuilder indexBuilder = new IndexBuilder("127.0.0.1",11211);
61 | 		Ad ad1 = new Ad();
62 | 		ad1.adId = 1125L;
63 | 		ad1.keyWords = new ArrayList<String>();
64 | 		ad1.keyWords.add("running");
65 | 		ad1.keyWords.add("shoe");
66 | 		ad1.keyWords.add("rebook");
67 | 		ad1.bidPrice = 4.5;
68 | 		ad1.campaignId = 99L;
69 | 		ad1.pClick = 0.34;
70 | 		indexBuilder.buildForwardIndex(ad1);
71 | 		try {
72 | 			MemcachedClient cache = new MemcachedClient(new InetSocketAddress("127.0.0.1",11211));
73 | 			@SuppressWarnings("unchecked")
74 | 			Ad  ad = (Ad)cache.get("1125");
75 | 			System.out.println("bid="+ad.bidPrice);
76 | 			System.out.println("pClick="+ad.pClick);
77 | 			System.out.println("campaignId="+ad.campaignId);
78 | 			int bid = (int) (ad.bidPrice * 100);
79 | 			assertEquals(bid , 450);
80 | 			assertEquals((long)ad.campaignId,99);
81 | 			int pClick = (int) (ad.pClick*100);
82 | 			assertEquals(pClick, 34);
83 | 		} catch (IOException e) {
84 | 			// TODO Auto-generated catch block
85 | 			e.printStackTrace();
86 | 		}
87 | 	}
88 | 
89 | }
90 | 


--------------------------------------------------------------------------------
/src/simpleads/io/bittiger/ads/AdsEngine.java:
--------------------------------------------------------------------------------
  1 | package simpleads.io.bittiger.ads;
  2 | 
  3 | import java.io.IOException;
  4 | import java.nio.charset.StandardCharsets;
  5 | import java.nio.file.Files;
  6 | import java.nio.file.Paths;
  7 | import java.util.ArrayList;
  8 | import java.util.List;
  9 | 
 10 | import org.json.*;
 11 | 
 12 | public class AdsEngine {
 13 | 	private String mAdsDataFilePath;
 14 | 	private String mBudgetFilePath;
 15 | 	private IndexBuilder indexBuilder;
 16 | 	private String mMemcachedServer;
 17 | 	private int mMemcachedPortal;
 18 | 	
 19 | 	public AdsEngine(String adsDataFilePath, String budgetDataFilePath,String memcachedServer,int memcachedPortal)
 20 | 	{
 21 | 		mAdsDataFilePath = adsDataFilePath;
 22 | 		mBudgetFilePath = budgetDataFilePath;
 23 | 		mMemcachedServer = memcachedServer;
 24 | 		mMemcachedPortal = memcachedPortal;
 25 | 		indexBuilder = new IndexBuilder(memcachedServer,memcachedPortal);
 26 | 	}
 27 | 	
 28 | 	public Boolean init()
 29 | 	{
 30 | 		try {
 31 | 			//load ads data
 32 | 			byte[] adsData;
 33 | 			adsData = Files.readAllBytes(Paths.get(mAdsDataFilePath));
 34 | 			String adsContent = new String(adsData, StandardCharsets.UTF_8);
 35 | 			JSONObject adsJsonObj = new JSONObject(adsContent);
 36 | 			JSONArray adsList = adsJsonObj.getJSONArray("ads");
 37 | 			for(int i = 0;i < adsList.length();i++)
 38 | 			{
 39 | 				Ad ad = new Ad();
 40 | 				JSONObject adJson = adsList.getJSONObject(i);				
 41 | 				ad.adId = adJson.getLong("adId");
 42 | 				ad.campaignId = adJson.getLong("campaignId");
 43 | 				ad.bidPrice = adJson.getDouble("bidPrice");
 44 | 				ad.pClick = adJson.getDouble("pClick");
 45 | 				ad.keyWords = new ArrayList<String>();
 46 | 				JSONArray keyWords = adJson.getJSONArray("keyWords");
 47 | 				for(int j = 0; j < keyWords.length();j++)
 48 | 				{
 49 | 					ad.keyWords.add(keyWords.getString(j));
 50 | 				}
 51 | 				
 52 | 				if(!indexBuilder.buildInvertIndex(ad) || !indexBuilder.buildForwardIndex(ad))
 53 | 				{
 54 | 					//log				
 55 | 				}				
 56 | 			}
 57 | 			
 58 | 			//load budget data
 59 | 			byte[] budgetData;
 60 | 			budgetData = Files.readAllBytes(Paths.get(mBudgetFilePath));
 61 | 			String budgetContent = new String(budgetData, StandardCharsets.UTF_8);
 62 | 			JSONObject budgetJsonObj = new JSONObject(budgetContent);
 63 | 			JSONArray campaignList = budgetJsonObj.getJSONArray("campaigns");
 64 | 			for(int i = 0 ; i < campaignList.length();i++)
 65 | 			{
 66 | 				JSONObject campaignJson = campaignList.getJSONObject(i);
 67 | 				Long campaignId = campaignJson.getLong("campaignId");
 68 | 				double budget = campaignJson.getDouble("budget");
 69 | 				if(!indexBuilder.updateBudget(campaignId, budget))
 70 | 				{
 71 | 					//log
 72 | 				}
 73 | 			}
 74 | 		} catch (IOException e) {
 75 | 			// TODO Auto-generated catch block
 76 | 			e.printStackTrace();
 77 | 		}
 78 | 		return true;
 79 | 	}
 80 | 	public List<Ad> selectAds(String query)
 81 | 	{
 82 | 		//query understanding
 83 | 		List<String> queryTerms = QueryParser.getInstance().QueryUnderstand(query);
 84 | 		//select ads candidates
 85 | 		List<Ad> adsCandidates = AdsSelector.getInstance(mMemcachedServer, mMemcachedPortal).selectAds(queryTerms);		
 86 | 		//L0 filter by pClick, relevance score
 87 | 		List<Ad> L0unfilteredAds = AdsFilter.getInstance().LevelZeroFilterAds(adsCandidates);
 88 | 		//rank 
 89 | 		List<Ad> rankedAds = AdsRanker.getInstance().rankAds(L0unfilteredAds);
 90 | 		//L1 filter by relevance score : select top K ads
 91 | 		int k = 20;
 92 | 		List<Ad> unfilteredAds = AdsFilter.getInstance().LevelOneFilterAds(rankedAds,k);
 93 | 		//Dedupe ads per campaign
 94 | 		List<Ad> dedupedAds = AdsCampaignManager.getInstance(mMemcachedServer, mMemcachedPortal).DedupeByCampaignId(unfilteredAds);
 95 | 		//pricing： next rank score/current score * current bid price
 96 | 		AdPricing.getInstance().setCostPerClick(dedupedAds);
 97 | 		//filter last one , ad without budget , ads with CPC < minReservePrice
 98 | 		List<Ad> ads = AdsCampaignManager.getInstance(mMemcachedServer, mMemcachedPortal).ApplyBudget(dedupedAds);
 99 | 		//allocation
100 | 		AdsAllocation.getInstance().AllocateAds(ads);
101 | 		return ads;
102 | 	}
103 | }
104 | 


--------------------------------------------------------------------------------