├── src ├── java │ └── README ├── resources │ ├── public │ │ ├── js │ │ │ └── common.js │ │ └── css │ │ │ └── common.css │ ├── messages │ │ ├── messages_en.properties │ │ └── messages.properties │ ├── spring-application-context.xml │ ├── cauca-logback.xml │ └── cauca-context.yaml └── clj │ └── cauca │ ├── api │ └── dao.clj │ ├── domain.clj │ ├── config.clj │ ├── log.clj │ ├── factory.clj │ ├── db.clj │ ├── util.clj │ ├── dao │ └── mysql_dao.clj │ └── component │ ├── rest.clj │ └── crawler.clj ├── img ├── cauca-01-home.png ├── cauca-02-api_courtauction.png └── cauca-03-api_courtauction_id.png ├── .gitignore ├── test └── clj │ └── cauca │ ├── component │ ├── crawler_test.clj │ └── rest_test.clj │ ├── util_test.clj │ └── dao │ └── mysql_dao_test.clj ├── .project ├── .classpath ├── docs └── cauca.sql ├── project.clj ├── README.md └── LICENSE /src/java/README: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/resources/public/js/common.js: -------------------------------------------------------------------------------- 1 | // TODO -------------------------------------------------------------------------------- /img/cauca-01-home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guriguri/cauca/HEAD/img/cauca-01-home.png -------------------------------------------------------------------------------- /img/cauca-02-api_courtauction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guriguri/cauca/HEAD/img/cauca-02-api_courtauction.png -------------------------------------------------------------------------------- /src/resources/public/css/common.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * TODO 3 | */ 4 | html, body { 5 | margin: 0; 6 | padding: 0; 7 | } -------------------------------------------------------------------------------- /img/cauca-03-api_courtauction_id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/guriguri/cauca/HEAD/img/cauca-03-api_courtauction_id.png -------------------------------------------------------------------------------- /src/clj/cauca/api/dao.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.api.dao 2 | (:use [cauca.domain]) 3 | ) 4 | 5 | (defprotocol courtauction-dao 6 | (add-courtauction [this courtauction]) 7 | (get-courtauction [this id]) 8 | (get-courtauction-list [this params]) 9 | ) -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /classes 2 | /.settings 3 | **/.settings 4 | .README* 5 | /bin 6 | /target 7 | **/target 8 | /lib 9 | /classes 10 | /checkouts 11 | pom.xml 12 | pom.xml.asc 13 | *.jar 14 | *.class 15 | .lein-deps-sum 16 | .lein-failures 17 | .lein-plugins 18 | .lein-repl-history 19 | *.swp 20 | *.log 21 | -------------------------------------------------------------------------------- /src/clj/cauca/domain.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.domain) 2 | 3 | (defstruct sigu :id :description) 4 | 5 | (defstruct courtauction 6 | :id :court :caNo :caDesc :itemNo 7 | :itemType :addr0 :addr1 :addr2 :addr 8 | :addrInfo :remarks :value :valueMin :auctionInfo 9 | :auctionTel :auctionDate :auctionLoc :status :regDate 10 | :updDate) -------------------------------------------------------------------------------- /src/resources/messages/messages_en.properties: -------------------------------------------------------------------------------- 1 | invalid.param.page=invalid parameter page 2 | invalid.param.pageSize=invalid parameter pageSize 3 | invalid.param.itemType=invalid parameter itemType 4 | invalid.param.addr0=invalid parameter addr0 5 | invalid.param.addr1=invalid parameter addr1 6 | invalid.param.auctionStartDate=invalid parameter auctionStartDate 7 | invalid.param.auctionEndDate=invalid parameter auctionEndDate 8 | page.not.found=page not found 9 | unknown.error=unknown error -------------------------------------------------------------------------------- /test/clj/cauca/component/crawler_test.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.component.crawler-test 2 | (:use [clojure test] 3 | ) 4 | (:require [cauca.factory :as f] 5 | [cauca.component.crawler :as crawler] 6 | [cauca.log :as log] 7 | [cauca.config :as config] 8 | ) 9 | ) 10 | 11 | (deftest add-courtauctions-test 12 | (log/configure-logback "/cauca-logback.xml") 13 | (config/config-yaml "/cauca-context.yaml") 14 | (let [dao-impl# (f/get-obj :courtauction-dao) 15 | sido (first (config/get-value :location.SidoCd))] 16 | (crawler/add-courtauctions! dao-impl# sido 20) 17 | ) 18 | ) -------------------------------------------------------------------------------- /src/resources/spring-application-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 8 | 9 | 10 | classpath:messages/messages 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | courtauction 4 | 5 | 6 | 7 | 8 | 9 | ccw.builder 10 | 11 | 12 | 13 | 14 | ccw.leiningen.builder 15 | 16 | 17 | 18 | 19 | org.eclipse.jdt.core.javabuilder 20 | 21 | 22 | 23 | 24 | 25 | org.eclipse.jdt.core.javanature 26 | ccw.leiningen.nature 27 | ccw.nature 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/clj/cauca/config.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.config 2 | (:use [clojure.java.io] 3 | [clj-yaml.core] 4 | [cauca.util])) 5 | 6 | (def yaml (ref nil)) 7 | 8 | (defn- read-yaml [path] 9 | (def url (get-url path)) 10 | (def sb (StringBuilder.)) 11 | (with-open [rdr (reader url)] 12 | (doseq [line (line-seq rdr)] 13 | (-> sb 14 | (.append line) 15 | (.append \newline)) 16 | ) 17 | ) 18 | (dosync 19 | (ref-set yaml (parse-string (.toString sb))) 20 | ) 21 | ) 22 | 23 | (defn config-yaml [path] 24 | (load-resource yaml read-yaml path)) 25 | 26 | (defn get-value [key] 27 | (when (nil? @yaml) 28 | (throw (IllegalAccessError. "Yaml Configuration Not Yet Loaded..."))) 29 | (let [parsed @yaml] 30 | (parsed key) 31 | ) 32 | ) -------------------------------------------------------------------------------- /test/clj/cauca/component/rest_test.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.component.rest-test 2 | (:use [clojure test] 3 | ) 4 | (:require [cauca.factory :as f] 5 | [cauca.component.rest :as rest] 6 | [cauca.log :as log] 7 | [cauca.config :as config] 8 | ) 9 | ) 10 | 11 | (defn request [method resource web-app & params] 12 | (web-app {:request-method method :uri resource :query-params (first params)})) 13 | 14 | (deftest test-routes 15 | (log/configure-logback "/cauca-logback.xml") 16 | (config/config-yaml "/cauca-context.yaml") 17 | (is (= 200 (:status (request :get "/" rest/main-routes)))) 18 | (is (= 200 (:status (request :get "/api/courtauction/691438" rest/main-routes)))) 19 | (is (= 200 (:status (request :get "/api/courtauction" rest/main-routes {:page 1 :pageSize 10}))))) -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/resources/messages/messages.properties: -------------------------------------------------------------------------------- 1 | invalid.param.page=page \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 2 | invalid.param.pageSize=pageSize \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 3 | invalid.param.itemType=itemType \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 4 | invalid.param.addr0=addr0 \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 5 | invalid.param.addr1=addr1 \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 6 | invalid.param.auctionStartDate=auctionStartDate \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 7 | invalid.param.auctionEndDate=auctionEndDate \ud30c\ub77c\ubbf8\ud130\uac00 \uc798\ubabb\ub418\uc5c8\uc2b5\ub2c8\ub2e4. 8 | page.not.found=\ud398\uc774\uc9c0\uac00 \uc5c6\uc2b5\ub2c8\ub2e4. 9 | unknown.error=\uc54c \uc218 \uc5c6\ub294 \uc5d0\ub7ec\uac00 \ubc1c\uc0dd\ud588\uc2b5\ub2c8\ub2e4. -------------------------------------------------------------------------------- /test/clj/cauca/util_test.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.util-test 2 | (:use [clojure test] 3 | [cauca.util] 4 | ) 5 | ) 6 | 7 | (deftest validation-test 8 | ; nil false 9 | (is (= false (validation nil 1 5 #"[a-z]+" true))) 10 | ; string true 11 | (is (= true (validation "abc" 1 5 #"[a-z]+" true))) 12 | ; string min false 13 | (is (= false (validation "abc" 4 5 #"[a-z]+" true))) 14 | ; string max false 15 | (is (= false (validation "abc" 1 2 #"[a-z]+" true))) 16 | ; string regex false 17 | (is (= false (validation "abc" 1 2 #"[a-b]+" true))) 18 | (is (= false (validation "abc" 1 2 #"[a-z]+" false))) 19 | ; number true 20 | (is (= true (validation 4 1 6 nil nil))) 21 | ; number min false 22 | (is (= false (validation 4 5 6 nil nil))) 23 | ; number max false 24 | (is (= false (validation 4 1 3 nil nil))) 25 | ) 26 | 27 | (deftest get-locale-test 28 | (is (not= nil (get-locale [{:headers {"accept-language" "ko-kr"}}]))) 29 | (is (not= nil (get-locale [nil]))) 30 | ) 31 | -------------------------------------------------------------------------------- /src/clj/cauca/log.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.log 2 | (:use [cauca.util]) 3 | (:require [clojure.tools [logging :as log]]) 4 | ) 5 | 6 | (defmacro log-message [& args] 7 | `(log/info (str ~@args))) 8 | 9 | (defmacro log-error [e & args] 10 | `(log/log :error ~e (str ~@args))) 11 | 12 | (defmacro log-debug [& args] 13 | `(log/debug (str ~@args))) 14 | 15 | (defmacro log-warn-error [e & args] 16 | `(log/warn (str ~@args) ~e)) 17 | 18 | (defmacro log-warn [& args] 19 | `(log/warn (str ~@args))) 20 | 21 | (defn log-capture! [& args] 22 | (apply log/log-capture! args)) 23 | 24 | (defn log-stream [& args] 25 | (apply log/log-stream args)) 26 | 27 | (defn configure-logback [path] 28 | (let [url (get-url path) 29 | lc (org.slf4j.LoggerFactory/getILoggerFactory) 30 | conf (ch.qos.logback.classic.joran.JoranConfigurator.)] 31 | (.reset lc) 32 | (.setContext conf lc) 33 | (.doConfigure conf url) 34 | (log-message "logback xml from classpath loaded...") 35 | ) 36 | ) -------------------------------------------------------------------------------- /docs/cauca.sql: -------------------------------------------------------------------------------- 1 | /* CREATE USER */ 2 | -- 3 | -- CREATE USER 'cauca'@'%' IDENTIFIED BY 'cauca.1234'; 4 | -- GRANT ALL PRIVILEGES ON cauca.* TO 'cauca'@'%' WITH GRANT option; 5 | -- FLUSH PRIVILEGES; 6 | -- 7 | /* CREATE DATABASE */ 8 | -- CREATE DATABASE cauca; 9 | 10 | CREATE TABLE courtauction ( 11 | id INT(11) NOT NULL AUTO_INCREMENT, 12 | court VARCHAR(16) NOT NULL, 13 | caNo VARCHAR(16) NOT NULL, 14 | caDesc TEXT, 15 | itemNo INT NOT NULL, 16 | itemType VARCHAR(16) NOT NULL, 17 | addr0 VARCHAR(16) NOT NULL, 18 | addr1 VARCHAR(16) NOT NULL, 19 | addr2 VARCHAR(16) NOT NULL, 20 | addr VARCHAR(100), 21 | addrInfo TEXT, 22 | remarks TEXT, 23 | value BIGINT NOT NULL, 24 | valueMin BIGINT NOT NULL, 25 | auctionInfo VARCHAR(64) NOT NULL, 26 | auctionTel VARCHAR(64) NOT NULL, 27 | auctionDate DATE NOT NULL, 28 | auctionLoc VARCHAR(64) NOT NULL, 29 | status VARCHAR(8) NOT NULL, 30 | regDate DATE NOT NULL, 31 | updDate DATE NOT NULL, 32 | PRIMARY KEY (id) 33 | ); 34 | 35 | CREATE UNIQUE INDEX COURTAUCTION_UK_01 ON courtauction (court, caNo, itemNo); -------------------------------------------------------------------------------- /src/clj/cauca/factory.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.factory 2 | (:use [cauca.util]) 3 | (:require [cauca.log :as log] 4 | [cauca.config :as config] 5 | [cauca.db :as db] 6 | [cauca.dao.mysql-dao :as dao] 7 | ) 8 | ) 9 | 10 | (def beans-map (ref nil)) 11 | 12 | (defn- beans [] 13 | (log/configure-logback "/cauca-logback.xml") 14 | (config/config-yaml "/cauca-context.yaml") 15 | (let [courtauction-dao (dao/mysql-courtauction-dao) 16 | spring-context (org.springframework.context.support.ClassPathXmlApplicationContext. (config/get-value :spring.contex.path)) 17 | message-source (.getBean spring-context "messageSource" org.springframework.context.MessageSource) 18 | add (fn [key obj] (alter beans-map assoc key obj))] 19 | (dosync 20 | (ref-set beans-map {}) 21 | (add :courtauction-dao courtauction-dao)) 22 | (add :spring-context spring-context) 23 | (add :message-source message-source) 24 | ) 25 | ) 26 | 27 | (defn get-obj [key] 28 | (if (nil? @beans-map) 29 | (load-resource beans-map beans)) 30 | (dosync 31 | (get @beans-map key) 32 | ) 33 | ) -------------------------------------------------------------------------------- /src/resources/cauca-logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | cauca.log 5 | 6 | UTF-8 7 | %date %level [%thread] %logger{10} %msg%n 8 | 9 | 10 | 11 | 12 | 13 | UTF-8 14 | %date %level [%thread] %logger{10} %msg%n 15 | 16 | 17 | 18 | 20 | cauca.log 21 | 22 | cauca.log.%d 23 | 7 24 | 25 | 26 | UTF-8 27 | %date %level [%thread] %logger{10} %msg%n 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/clj/cauca/db.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.db 2 | (:require [cauca.config :as config] 3 | ) 4 | (:import [com.mchange.v2.c3p0 ComboPooledDataSource] 5 | ) 6 | ) 7 | 8 | (defn pooled-data-source [] 9 | (config/config-yaml "/cauca-context.yaml") 10 | (let [db {:classname (config/get-value :db.classname) 11 | :subprotocol (config/get-value :db.subprotocol) 12 | :subname (config/get-value :db.subname) 13 | :user (config/get-value :db.user) 14 | :password (config/get-value :db.password) 15 | :db.max.idle.time.excess.connection.sec 16 | (config/get-value :db.max.idle.time.excess.connection.sec) 17 | :db.max.idel.time.sec (config/get-value :db.max.idel.time.sec) } 18 | datasource (ComboPooledDataSource.)] 19 | (.setDriverClass datasource (:classname db)) 20 | (.setJdbcUrl datasource (str "jdbc:" (:subprotocol db) ":" (:subname db))) 21 | (.setUser datasource (:user db)) 22 | (.setPassword datasource (:password db)) 23 | (.setMaxIdleTimeExcessConnections datasource 24 | (:db.max.idle.time.excess.connection.sec db)) 25 | (.setMaxIdleTime datasource (:db.max.idel.time.sec db)) 26 | {:datasource datasource} 27 | ) 28 | ) 29 | 30 | (def connection-pool 31 | (delay (pooled-data-source)) 32 | ) 33 | 34 | (defn connection [] @connection-pool) -------------------------------------------------------------------------------- /test/clj/cauca/dao/mysql_dao_test.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.dao.mysql-dao-test 2 | (:use [clojure test] 3 | [cauca.domain] 4 | ) 5 | (:require [cauca.factory :as f] 6 | ) 7 | ) 8 | 9 | (deftest add-courtauction-test 10 | (let [dao-impl# (f/get-obj :courtauction-dao) 11 | a-courtauction (struct courtauction 12 | nil "court" (.getTime (new java.util.Date)) 13 | "caDesc" 1 "itemType" 14 | "addr0" "addr1" "addr2" 15 | "addr" "addrInfo" "remarks" 16 | 2000000 1000000 "auctionInfo" 17 | "auctionTel" "2013-10-01 10:00" "auctionLoc" 18 | "status" (new java.util.Date) (new java.util.Date)) 19 | id (:generated_key (first (.add-courtauction dao-impl# a-courtauction))) 20 | ret-courtauction (first (.get-courtauction dao-impl# id))] 21 | (println (:caNo a-courtauction) (:caNo ret-courtauction)) 22 | (is (:caNo a-courtauction) (:caNo ret-courtauction)) 23 | ) 24 | ) 25 | 26 | (deftest get-courtauction-list-test 27 | (let [dao-impl# (f/get-obj :courtauction-dao) 28 | ret-courtauction-list (.get-courtauction-list dao-impl# 29 | {"itemType" "아파트,임야" 30 | "addr0" "서울특별시,경기도" 31 | "addr1" "강남구,군포시" 32 | "page" 0 33 | "pageSize" 10})] 34 | (println (str "count=" (count ret-courtauction-list))) 35 | (is (> (count ret-courtauction-list) 0)))) -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject cauca "0.1.0-SNAPSHOT" 2 | :url "http://github.com/guriguri/cauca" 3 | :license {:name "Apache License, Version 2.0" :url "http://www.apache.org/licenses/LICENSE-2.0"} 4 | :source-paths ["src/clj"] 5 | :java-source-paths ["src/java"] 6 | :test-paths ["test/clj"] 7 | :resource-paths ["src/resources"] 8 | :compile-path "target/classes" 9 | :target-path "target/" 10 | 11 | :dependencies [[org.clojure/clojure "1.4.0"] 12 | [junit "4.8"] 13 | [org.clojure/tools.logging "0.2.3"] 14 | [org.clojure/data.json "0.2.3"] 15 | [ch.qos.logback/logback-classic "1.0.6"] 16 | [org.slf4j/log4j-over-slf4j "1.6.6"] 17 | [clj-yaml "0.4.0"] 18 | [clj-http "0.7.2"] 19 | [enlive "1.0.0"] 20 | [org.clojure/java.jdbc "0.3.0-alpha5"] 21 | [mysql/mysql-connector-java "5.1.25"] 22 | [c3p0/c3p0 "0.9.1.2"] 23 | [compojure "1.1.3"] 24 | [hiccup "0.3.6"] 25 | [ring/ring-devel "0.3.11"] 26 | [ring/ring-jetty-adapter "0.3.11"] 27 | [org.springframework/spring-context "3.1.2.RELEASE"] 28 | ] 29 | 30 | :plugins [[lein-daemon "0.5.4"]] 31 | 32 | :repositories [["java.net" "http://download.java.net/maven/2"] 33 | ["conjars" "http://conjars.org/repo"] 34 | ] 35 | 36 | :jar-exclusions [#"cauca-logback.xml" 37 | #"cauca-context.yaml"] 38 | 39 | :daemon {:crawler {:ns cauca.component.crawler :pidfile "crawler.pid"} 40 | :rest {:ns cauca.component.rest :pidfile "rest.pid"} 41 | } 42 | 43 | :aot [cauca.component.crawler 44 | cauca.component.rest 45 | ] 46 | ) 47 | -------------------------------------------------------------------------------- /src/clj/cauca/util.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.util 2 | (:require [clojure.string :as string] 3 | ) 4 | ) 5 | 6 | (defmacro load-resource [ref-var & init-fn] 7 | `(dosync 8 | (when (nil? @~ref-var)) 9 | (~@init-fn))) 10 | 11 | (deftype Foo []) 12 | (defn get-url [path] (.getResource (.getClass (Foo.)) path)) 13 | 14 | (defn validation [value min max regex isMatches] 15 | (if (nil? value) 16 | false 17 | (if (string? value) 18 | (if (and (= (nil? min) false) (< (count value) min)) 19 | false 20 | (if (and (= (nil? max) false) (> (count value) max)) 21 | false 22 | (if (= (nil? regex) false) 23 | (let [groups (re-matches regex value)] 24 | (if (= (nil? groups) isMatches) false true)) 25 | ) 26 | ) 27 | ) 28 | (if (and (= (nil? min) false) (< value min)) 29 | false 30 | (if (and (= (nil? max) false) (> value max)) false true) 31 | ) 32 | ) 33 | ) 34 | ) 35 | 36 | (defn get-locale [request] 37 | (let [locale (ref nil)] 38 | (if (and (= (nil? request) false) (= (nil? (first request)) false)) 39 | (dosync 40 | (doseq [[lang, country, variant] 41 | (map #(. % split "_") 42 | (map #(string/replace % #"-" "_") 43 | (. (((first request) :headers) "accept-language") split ",; ")))] 44 | (if (nil? country) 45 | (ref-set locale (new java.util.Locale lang)) 46 | (if (nil? variant) 47 | (ref-set locale (new java.util.Locale lang country)) 48 | (ref-set locale (new java.util.Locale lang country variant)) 49 | ) 50 | ) 51 | ) 52 | ) 53 | ) 54 | (if (nil? @locale) 55 | (java.util.Locale/getDefault) 56 | @locale) 57 | ) 58 | ) -------------------------------------------------------------------------------- /src/resources/cauca-context.yaml: -------------------------------------------------------------------------------- 1 | ########## These all have default values as shown 2 | 3 | db.classname: "com.mysql.jdbc.Driver" 4 | db.subprotocol: "mysql" 5 | db.subname: "//cauca:3306/cauca" 6 | db.user: "cauca" 7 | db.password: "cauca.1234" 8 | db.max.idle.time.excess.connection.sec: 1800 9 | db.max.idel.time.sec: 10800 10 | 11 | port: 8080 12 | spring.contex.path: "classpath:/spring-application-context.xml" 13 | 14 | location.SidoCd: 15 | - 11, 서울특별시 16 | - 26, 부산광역시 17 | - 27, 대구광역시 18 | - 28, 인천광역시 19 | - 29, 광주광역시 20 | - 30, 대전광역시 21 | - 31, 울산광역시 22 | - 36, 세종특별자치시 23 | - 41, 경기도 24 | - 42, 강원도 25 | - 43, 충청북도 26 | - 44, 충청남도 27 | - 45, 전라북도 28 | - 46, 전라남도 29 | - 47, 경상북도 30 | - 48, 경상남도 31 | - 50, 제주특별자치도 32 | 33 | #location.SiguCd: 34 | # - 820, 가평군 35 | # - 281, 고양시 덕양구 36 | # - 471, 고양시 덕양구 37 | # - 473, 고양시 일산구 38 | # - 285, 고양시 일산동구 39 | # - 287, 고양시 일산서구 40 | # - 290, 과천시 41 | # - 210, 광명시 42 | # - 790, 광주군 43 | # - 610, 광주시 44 | # - 310, 구리시 45 | # - 410, 군포시 46 | # - 570, 김포시 47 | # - 360, 남양주시 48 | # - 250, 동두천시 49 | # - 190, 부천시 50 | # - 197, 부천시 소사구 51 | # - 199, 부천시 오정구 52 | # - 195, 부천시 원미구 53 | # - 135, 성남시 분당구 54 | # - 131, 성남시 수정구 55 | # - 133, 성남시 중원구 56 | # - 113, 수원시 권선구 57 | # - 117, 수원시 영통구 58 | # - 111, 수원시 장안구 59 | # - 115, 수원시 팔달구 60 | # - 390, 시흥시 61 | # - 273, 안산시 단원구 62 | # - 271, 안산시 상록구 63 | # - 860, 안성군 64 | # - 550, 안성시 65 | # - 173, 안양시 동안구 66 | # - 171, 안양시 만안구 67 | # - 630, 양주시 68 | # - 830, 양평군 69 | # - 730, 여주군 70 | # - 800, 연천군 71 | # - 370, 오산시 72 | # - 490, 용인시 73 | # - 463, 용인시 기흥구 74 | # - 465, 용인시 수지구 75 | # - 461, 용인시 처인구 76 | # - 430, 의왕시 77 | # - 150, 의정부시 78 | # - 530, 이천시 79 | # - 500, 이천시 80 | # - 480, 파주시 81 | # - 510, 파주시 82 | # - 220, 평택시 83 | # - 650, 포천시 84 | # - 450, 하남시 85 | # - 750, 화성군 86 | # - 590, 화성시 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CAUCA 2 | 3 | ## 소개 4 | * CAUCA는 Court AUCtion Api service로 [clojure](http://clojure.org)를 이용해서 법원 경매 사이트를 수집하고, 사용자에게 REST API를 제공하는 서비스입니다. 5 | 6 | ## 요구사항 7 | * JDK 6+ 8 | * MySQL 5.x 9 | * [leiningen](http://leiningen.org) 10 | 11 | ``` 12 | $> wget --no-check-certificate https://raw.github.com/technomancy/leiningen/stable/bin/lein 13 | $> mv lein ~/bin/ 14 | $> chmod 755 ~/bin/lein 15 | $> lein self-install 16 | ``` 17 | 18 | ## 환경설정 19 | * MySQL 20 | * USER 생성 21 | 22 | ``` 23 | mysql> CREATE USER '$CAUCA_USER$'@'%' IDENTIFIED BY '$CAUCA_PASSWORD$'; 24 | mysql> GRANT ALL PRIVILEGES ON cauca.* TO '$CAUCA_USER$'@'%' WITH GRANT option; 25 | mysql> FLUSH PRIVILEGES; 26 | ``` 27 | * DATABASE 생성 28 | 29 | ``` 30 | mysql> CREATE DATABASE cauca; 31 | mysql> FLUSH PRIVILEGES; 32 | ``` 33 | * src/resources/cauca-context.yaml 에서 DB 정보 수정 34 | 35 | ``` 36 | : 37 | db.subname: "//$CAUCA_HOST$:3306/cauca" 38 | db.user: "$CAUCA_USER$" 39 | db.password: "$CAUCA_PASSWORD$" 40 | : 41 | ``` 42 | 43 | ## 실행방법 44 | 45 | ``` 46 | # build 47 | $> lein do clean, uberjar 48 | 49 | # crawler 50 | ## crawler start 51 | ## Usage: lein daemon start crawler [REPEAT COUNT] [SLEEP SECOND] 52 | ## REPEAT COUNT: -1(infinite), 1, 2, ... 53 | ## SLEEP SECOND: Repeat the cycle 54 | $> lein daemon start crawler -1 86400 55 | ## crawler stop 56 | $> lein daemon stop crawler 57 | 58 | # rest 59 | ## rest start 60 | $> lein daemon start rest 61 | ## test main-page: http://127.0.0.1:8080 62 | ## rest stop 63 | $> lein daemon stop rest 64 | 65 | # lein daemon 은 Unix 계열에서만 실행이 되므로 Windows 에서는 uberjar 로 만든 jar 파일을 직접 실행하면 됩니다. 66 | $> java -cp .;src\resources;target\cauca-0.1.0-SNAPSHOT-standalone.jar cauca.component.crawler [REPEAT COUNT] [SLEEP SECOND] 67 | $> java -cp .;src\resources;target\cauca-0.1.0-SNAPSHOT-standalone.jar cauca.component.rest 68 | ``` 69 | 70 | ## Demo 71 | * Screenshot 72 | * home ![Alt Text](./img/cauca-01-home.png "home") 73 | * /api/countauction ![Alt Text](./img/cauca-02-api_courtauction.png "/api/courtauction") 74 | * /api/countauction/:id ![Alt Text](./img/cauca-03-api_courtauction_id.png "/api/courtauction/:id") -------------------------------------------------------------------------------- /src/clj/cauca/dao/mysql_dao.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.dao.mysql-dao 2 | (:use [cauca.api.dao] 3 | [cauca.domain] 4 | ) 5 | (:require [cauca.db :as db-pool] 6 | [clojure.java.jdbc :as jdbc] 7 | [clojure.java.jdbc.sql :as sql] 8 | [clojure.string :as string] 9 | ) 10 | ) 11 | 12 | (defn get-query-str [params] 13 | (let [getNum (fn [data default] (if (nil? data) default (if (string? data) (Integer/parseInt data) data))) 14 | getVal (fn [data min max] (if (< data min) min (if (> data max) max data))) 15 | page (getVal (getNum (params "page") 1) 1 100) 16 | pageSize (getVal (getNum (params "pageSize") 10) 1 100)] 17 | (string/join \newline 18 | ["SELECT * FROM courtauction" 19 | "WHERE 1 = 1" 20 | (string/join 21 | \newline 22 | (map #(str "AND " (key %) " IN ( '" (string/replace (val %) #"," "','") "') ") 23 | (filter #(contains? #{"itemType" "addr0" "addr1"} (key %)) params))) 24 | (if-not (nil? (params "minValue")) (str "AND valueMin >= " (params "minValue"))) 25 | (if-not (nil? (params "maxValue")) (str "AND valueMin <= " (params "maxValue"))) 26 | (if-not (nil? (params "auctionStartDate")) (str "AND auctionDate >= '" (params "auctionStartDate") "'")) 27 | (if-not (nil? (params "auctionEndDate")) (str "AND auctionDate <= '" (params "auctionEndDate") "'")) 28 | "ORDER BY id DESC" 29 | (str "LIMIT " (* (- page 1) pageSize) ", " pageSize)]))) 30 | 31 | (defn mysql-courtauction-dao [] 32 | (reify 33 | courtauction-dao 34 | (add-courtauction [this courtauction] 35 | (jdbc/insert! (db-pool/connection) :courtauction {:court (:court courtauction) 36 | :caNo (:caNo courtauction) 37 | :caDesc (:caDesc courtauction) 38 | :itemNo (:itemNo courtauction) 39 | :itemType (:itemType courtauction) 40 | :addr0 (:addr0 courtauction) 41 | :addr1 (:addr1 courtauction) 42 | :addr2 (:addr2 courtauction) 43 | :addr (:addr courtauction) 44 | :addrInfo (:addrInfo courtauction) 45 | :remarks (:remarks courtauction) 46 | :value (:value courtauction) 47 | :valueMin (:valueMin courtauction) 48 | :auctionInfo (:auctionInfo courtauction) 49 | :auctionTel (:auctionTel courtauction) 50 | :auctionDate (:auctionDate courtauction) 51 | :auctionLoc (:auctionLoc courtauction) 52 | :status (:status courtauction) 53 | :regDate (:regDate courtauction) 54 | :updDate (:updDate courtauction) 55 | } 56 | ) 57 | ) 58 | (get-courtauction [this id] 59 | (jdbc/query (db-pool/connection) 60 | (sql/select * :courtauction (sql/where {:id id})) 61 | :identifiers str 62 | ) 63 | ) 64 | (get-courtauction-list [this params] 65 | (jdbc/query (db-pool/connection) 66 | [(get-query-str params)] 67 | :identifiers str 68 | ) 69 | ) 70 | ) 71 | ) -------------------------------------------------------------------------------- /src/clj/cauca/component/rest.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.component.rest 2 | (:use [compojure.core] 3 | [ring.middleware.reload] 4 | [hiccup core page-helpers] 5 | [ring.adapter.jetty :only [run-jetty]] 6 | ) 7 | (:require [cauca.factory :as f] 8 | [cauca.log :as log] 9 | [cauca.util :as util] 10 | [cauca.config :as config] 11 | [compojure.route :as route] 12 | [compojure.handler :as handler] 13 | [clojure.data.json :as json] 14 | [ring.util.response :as resp] 15 | ) 16 | (:gen-class) 17 | ) 18 | 19 | (defn main-page [] 20 | (html5 21 | [:head 22 | [:title "CAUCA"] 23 | (include-css "/css/common.css") 24 | (include-js "/js/common.js") 25 | ] 26 | [:body 27 | [:h1 "CAUCA API"] 28 | [:ul 29 | [:li (link-to "/api/courtauction" "/api/courtauction") 30 | [:ul 31 | [:li "법원 경매 정보 리스트 조회"] 32 | [:li "method:get"] 33 | [:li "parameter" 34 | [:ul 35 | [:li "itemType: 경매 물건의 유형(근린시설, 기타, 다가구주택, 다세대, 단독주택, 대지, 빌라, 상가, 아파트, 연립주택, 오피스텔, 임야, 자동차, 전답, 중기 중 1개 선택)"] 36 | [:li "addr0: 시도 (서울특별시, 부산광역시, 대구광역시, 인천광역시, 광주광역시, 대전광역시, 울산광역시, 세종특별자치시, 경기도, 강원도, 충청북도, 충청남도, 전라북도, 전라남도, 경상북도, 경상남도, 제주특별자치도 중 1개 선택)"] 37 | [:li "addr1: 구군 (강남구, 은평구, 과천시, 고양시 등)"] 38 | [:li "minValue: 최소 경매가 (단위: 원)"] 39 | [:li "maxValue: 최대 경매가 (단위: 원)"] 40 | [:li "auctionStartDate: 조회하고 싶은 경매일 구간의 시작 (포멧: yyyy-mm-dd)"] 41 | [:li "auctionEndDate: 조회하고 싶은 경매일 구간의 종료 (포멧: yyyy-mm-dd)"] 42 | [:li "page: 조회하고 싶은 페이지 구간 (1-100, default: 1)"] 43 | [:li "pageSize: 한 페이지에 표시하고 싶은 경매정보 수 (1-100, default: 10)"] 44 | ] 45 | ] 46 | ] 47 | ] 48 | [:li "/api/courtauction/:id" 49 | [:ul 50 | [:li "법원 경매 정보 조회"] 51 | [:li "method:get"] 52 | [:li "parameter" 53 | [:ul 54 | [:li "id: 경매 물건 ID"] 55 | ] 56 | ] 57 | ] 58 | ] 59 | ] 60 | ] 61 | ) 62 | ) 63 | 64 | (defn get-msg [args & request] 65 | (let [msg-source (f/get-obj :message-source) 66 | locale (util/get-locale request)] 67 | (try 68 | (.getMessage msg-source (last args), (to-array args), locale) 69 | (catch Exception e 70 | (do 71 | (log/log-error e) 72 | (.getMessage msg-source "unknown.error", (to-array args), locale) 73 | ) 74 | ) 75 | ) 76 | ) 77 | ) 78 | 79 | (defn cauca-writer [key value] 80 | (if (or (= key :auctionDate) (= key :regDate) (= key :updDate)) 81 | (str (java.sql.Date. (.getTime value))) 82 | value 83 | ) 84 | ) 85 | 86 | (defn response [uri json status] 87 | (log/log-message "uri=" uri ", json=" json) 88 | (-> (resp/response json) 89 | (resp/status status) 90 | (resp/header "Access-Control-Allow-Origin" "*") 91 | (resp/content-type "application/json; charset=utf-8") 92 | ) 93 | ) 94 | 95 | (defn check-validation [value min max regex match? msg-id] 96 | (if-not (nil? value) 97 | (if (false? (util/validation value min max regex match?)) 98 | (throw (Exception. msg-id)) 99 | ) 100 | ) 101 | ) 102 | 103 | (defn check-params [params] 104 | (check-validation (params "page") 1 3 #"[0-9]+" true "invalid.param.page") 105 | (check-validation (params "pageSize") 1 3 #"[0-9]+" true "invalid.param.pageSize") 106 | (check-validation (params "itemType") nil nil #".*[\s~!@#$%^&*()_+`\\=\-{}|\\[\\]:\\\\\";'<>?./].*" false "invalid.param.itemType") 107 | (check-validation (params "addr0") nil nil #".*[\s~!@#$%^&*()_+`\\=\-{}|\\[\\]:\\\\\";'<>?./].*" false "invalid.param.addr0") 108 | (check-validation (params "addr1") nil nil #".*[\s~!@#$%^&*()_+`\\=\-{}|\\[\\]:\\\\\";'<>?./].*" false "invalid.param.addr1") 109 | (check-validation (params "auctionStartDate") 9 11 #"20[0-9][0-9]-[0-1][0-9]-[0-3][0-9]" true "invalid.param.auctionStartDate") 110 | (check-validation (params "auctionEndDate") 9 11 #"20[0-9][0-9]-[0-1][0-9]-[0-3][0-9]" true "invalid.param.auctionEndDate") 111 | ) 112 | 113 | (defn get-json [map-obj] 114 | (json/write-str map-obj :value-fn cauca-writer :escape-unicode false) 115 | ) 116 | 117 | (defroutes main-routes 118 | (GET "/" [] (main-page)) 119 | (GET ["/api/courtauction/:id", :id #"[0-9]+"] [id :as request] 120 | (let [dao-impl# (f/get-obj :courtauction-dao) 121 | ret-courtauction (first (.get-courtauction dao-impl# id)) 122 | json (get-json {:msg "ok" :result ret-courtauction})] 123 | (response (request :uri) json 200))) 124 | ; (GET "/api/courtauction" {params :query-params} 125 | (GET "/api/courtauction" request 126 | ; (log/log-message request) 127 | (check-params (request :query-params)) 128 | (let [dao-impl# (f/get-obj :courtauction-dao) 129 | params (request :query-params) 130 | ret-courtauction-list (.get-courtauction-list dao-impl# params) 131 | json (get-json {:msg "ok" :result ret-courtauction-list})] 132 | (response (request :uri) json 200))) 133 | (route/resources "/") 134 | (route/not-found 135 | {:headers {"Content-Type" "application/json; charset=utf-8"} 136 | :body (get-json {:msg (get-msg ["page.not.found"])}) 137 | }) 138 | ) 139 | 140 | (def main-handler 141 | (-> main-routes handler/api)) 142 | 143 | (defn catch-errors [handler] 144 | (fn [request] 145 | (try 146 | (handler request) 147 | (catch Exception e 148 | (log/log-error e) 149 | (let [json (get-json {:msg (get-msg [(.getMessage e)] request)})] 150 | (response (request :uri) json 400)) 151 | ) 152 | ) 153 | ) 154 | ) 155 | 156 | (def app 157 | (-> #'main-handler 158 | (wrap-reload '[cauca.component.rest]) 159 | catch-errors)) 160 | 161 | (defn start-server! [] 162 | (config/config-yaml "/cauca-context.yaml") 163 | (run-jetty app {:port (config/get-value :port) :join? false}) 164 | ) 165 | 166 | (defn -main [] 167 | (start-server!) 168 | ) -------------------------------------------------------------------------------- /src/clj/cauca/component/crawler.clj: -------------------------------------------------------------------------------- 1 | (ns cauca.component.crawler 2 | (:use [cauca.domain] 3 | ) 4 | (:require [cauca.factory :as f] 5 | [cauca.log :as log] 6 | [cauca.config :as config] 7 | [clj-http.client :as client] 8 | [net.cgrand.enlive-html :as e] 9 | [clojure.string :as string] 10 | ) 11 | (:gen-class) 12 | ) 13 | 14 | (defn get-value[coll key] 15 | (if (map? coll) 16 | (key coll) 17 | coll 18 | ) 19 | ) 20 | 21 | (defn trim[data idx] 22 | (if (nil? data) 23 | data 24 | (if (seq? data) 25 | (string/trim (nth data idx)) 26 | (string/trim data)) 27 | ) 28 | ) 29 | 30 | (defn get-addr-info [addr-info] 31 | (let [addr (first (:content (nth addr-info 1))) 32 | area (last addr-info)] 33 | (map #(string/trim (string/replace % #"\n" " ")) (seq [addr area])) 34 | ) 35 | ) 36 | 37 | (defn get-auction-info [auction-info] 38 | (let [auction-agent (first auction-info) 39 | info (re-seq #"'[^']+'" (:onclick (:attrs (second auction-info)))) 40 | auction-tel (first info) 41 | auction-date (second info) 42 | auction-salesroom (nth info 2)] 43 | (map #(string/trim (string/replace % #"'" "")) 44 | (seq [auction-agent auction-tel auction-date auction-salesroom])) 45 | ) 46 | ) 47 | 48 | (defn set-courtauction [cols] 49 | (let [caInfo (map #(trim % 0) 50 | (filter #(not-empty (trim % 0)) 51 | (map #(get-value % :content) 52 | (:content (first (:content (nth cols 1))))))) 53 | itemInfo (map #(string/trim %) 54 | (filter #(= (map? %) false) (:content (nth cols 2)))) 55 | addrInfo (get-addr-info 56 | (:content (first 57 | (e/select-nodes* 58 | (:content (nth cols 3)) [(e/tag= :div)])))) 59 | addrs (map #(string/trim %) 60 | (filter #(not-empty %) (. (first addrInfo) split " "))) 61 | remarks (map #(string/trim 62 | (string/replace % #"\n" " ")) (:content (nth cols 4))) 63 | valueInfo (map #(string/trim (first (:content %))) 64 | (e/select-nodes* (:content (nth cols 5)) [(e/tag= :div)])) 65 | auctionInfo (get-auction-info 66 | (:content (first 67 | (e/select-nodes* 68 | (:content (nth cols 6)) [(e/tag= :div)])))) 69 | status (map #(string/trim %) 70 | (filter #(= (map? %) false) (:content (nth cols 7)))) 71 | now (new java.util.Date) 72 | ] 73 | (struct courtauction 74 | nil 75 | (nth caInfo 0) ;:court 76 | (nth caInfo 1) ;:caNo 77 | (string/join " " caInfo) ;:caDesc 78 | (nth itemInfo 0) ;:itemNo 79 | (nth itemInfo 1) ;:itemType 80 | (nth addrs 0) ;:addr0 81 | (nth addrs 1) ;:addr1 82 | (nth addrs 2) ;:addr2 83 | (string/join " " addrs) ;:addr 84 | (nth addrInfo 1) ;:addrInfo 85 | (nth remarks 0);:remarks 86 | (string/replace (nth valueInfo 0) "," "") ;:value 87 | (string/replace (nth valueInfo 1) "," "") ;:valueMin 88 | (nth auctionInfo 0) ;:auctionInfo 89 | (nth auctionInfo 1) ;:auctionTel 90 | (nth auctionInfo 2) ;:auctionDate 91 | (nth auctionInfo 3) ;:auctionLoc 92 | (nth status 0) ;:status 93 | now ;:regDate 94 | now ;:updDate 95 | ) 96 | ) 97 | ) 98 | 99 | (defn courtauction-parser [html] 100 | (let [nodes (e/html-snippet html) 101 | courtauctions (ref [])] 102 | (doseq [rows (e/select-nodes* nodes [(e/attr= :class "Ltbl_list") 103 | (e/tag= :tbody) 104 | (e/tag= :tr)])] 105 | (try 106 | (dosync (alter courtauctions conj 107 | (set-courtauction 108 | (e/select-nodes* (:content rows) [(e/tag= :td)])))) 109 | (catch Exception e 110 | (log/log-error e (e/select-nodes* (:content rows) [(e/tag= :td)])) 111 | ) 112 | ) 113 | ) 114 | courtauctions 115 | ) 116 | ) 117 | 118 | (defn sigu-parser [xml] 119 | (let [nodes (e/html-snippet xml) 120 | sigus (ref [])] 121 | (doseq [row (filter #(not-empty (:value (:attrs %))) 122 | (e/select-nodes* nodes [(e/tag= :option)]))] 123 | (dosync (alter sigus conj 124 | (struct sigu (:value (:attrs row)) 125 | (first (map #(string/trim %) (:content row)))))) 126 | ) 127 | sigus 128 | ) 129 | ) 130 | 131 | (defn get-sigu-list! [sido] 132 | (let [resp (client/post 133 | "https://www.courtauction.go.kr/RetrieveAucSigu.ajax" 134 | { 135 | :as "euc-kr" 136 | :headers { "Host" "www.courtauction.go.kr" 137 | "User-Agent" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.8; ko; rv:1.9.0.14) Gecko/2009082706 Firefox/3.0.14" 138 | "Accept" "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" 139 | "Accept-Language" "ko-KR,ko;q=0.8,en-US;q=0.5,en;q=0.3" 140 | "Accept-Charset" "windows-949,utf-8;q=0.7,*;q=0.7" 141 | } 142 | :form-params { 143 | :sidoCode sido 144 | :id2 "idSiguCode" 145 | :id3 "idDongCode" 146 | } 147 | } 148 | ) ] 149 | (sigu-parser (:body resp)) 150 | ) 151 | ) 152 | 153 | (defn get-auction-list! [sido sigu page-size] 154 | (let [all-courtauctions (ref []) 155 | courtauctions (ref nil) 156 | resp (ref nil)] 157 | (loop [target-row 1] 158 | (dosync 159 | (ref-set resp (client/post "https://www.courtauction.go.kr/RetrieveRealEstMulDetailList.laf" 160 | {:as "euc-kr" 161 | :headers { "Host" "www.courtauction.go.kr" 162 | "User-Agent" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.8; ko; rv:1.9.0.14) Gecko/2009082706 Firefox/3.0.14" 163 | "Accept" "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" 164 | "Accept-Language" "ko-KR,ko;q=0.8,en-US;q=0.5,en;q=0.3" 165 | "Accept-Charset" "windows-949,utf-8;q=0.7,*;q=0.7" 166 | } 167 | :form-params { 168 | :_FORM_YN "Y" 169 | :bubwLocGubun "2" 170 | :daepyoSidoCd sido 171 | :daepyoSiguCd sigu 172 | :mDaepyoSidoCd sido 173 | :mDaepyoSiguCd sigu 174 | :srnID "PNO102000" 175 | :targetRow target-row 176 | } 177 | }) 178 | ) 179 | (ref-set courtauctions @(courtauction-parser (:body @resp))) 180 | (alter all-courtauctions into @courtauctions) 181 | ) 182 | (if (= (count @courtauctions) page-size) 183 | (recur (+ target-row page-size)) 184 | all-courtauctions 185 | ) 186 | ) 187 | ) 188 | ) 189 | 190 | (defn add-courtauctions! [dao-impl# sido page-size] 191 | (let [sido-code (first (. sido split ",")) 192 | courtauctions (ref nil)] 193 | (doseq [sigu @(get-sigu-list! sido-code)] 194 | (dosync 195 | (ref-set courtauctions 196 | @(get-auction-list! sido-code (:id sigu) page-size)) 197 | ) 198 | (doseq [courtauction @courtauctions] 199 | (try 200 | (.add-courtauction dao-impl# courtauction) 201 | (catch Exception e 202 | (if (or 203 | (= (instance? com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException e) false) 204 | (not= (.getErrorCode e) 1062)) 205 | (log/log-error e courtauction) 206 | ) 207 | ) 208 | ) 209 | ) 210 | (log/log-message "sido=" sido ", sigu=" sigu 211 | ", rows.count=" (count @courtauctions)) 212 | (Thread/sleep 1000) 213 | ) 214 | ) 215 | ) 216 | 217 | (defn -launch 218 | [repeatCnt sleepMsec] 219 | (log/configure-logback "/cauca-logback.xml") 220 | (config/config-yaml "/cauca-context.yaml") 221 | (log/log-message "START crawler!!!, repeatCnt=" repeatCnt ", sleepMsec=" sleepMsec) 222 | (loop [dao-impl# (f/get-obj :courtauction-dao) cnt 1] 223 | (doseq [sido (config/get-value :location.SidoCd)] 224 | (add-courtauctions! dao-impl# sido 20) 225 | ) 226 | (log/log-message "currentCnt=" cnt) 227 | (if (or (= repeatCnt -1) (< cnt repeatCnt)) 228 | (do 229 | (Thread/sleep sleepMsec) 230 | (recur (f/get-obj :courtauction-dao) (inc cnt)) 231 | ) 232 | ) 233 | ) 234 | (log/log-message "STOP crawler!!!") 235 | ) 236 | 237 | (defn -main 238 | [& args] 239 | (let [argsCnt (count args) 240 | repeatCnt (ref 1) 241 | sleepMsec (ref 1000)] 242 | (log/log-message "\n\nUsage: crawler [REPEAT COUNT] [SLEEP SECOND]\n REPEAT COUNT: -1(infinite), 1, 2, ...\n SLEEP SECOND: Repeat the cycle\n") 243 | (dosync 244 | (if (= argsCnt 1) 245 | (ref-set repeatCnt (Integer/parseInt (first args))) 246 | (if (>= argsCnt 2) 247 | (do 248 | (ref-set repeatCnt (Integer/parseInt (first args))) 249 | (ref-set sleepMsec (* (Integer/parseInt (second args)) 1000)) 250 | ) 251 | ) 252 | ) 253 | ) 254 | (if (= repeatCnt -1) 255 | (.start (Thread. -launch @repeatCnt @sleepMsec)) 256 | (-launch @repeatCnt @sleepMsec) 257 | ) 258 | ) 259 | ) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------