├── pictures ├── 流程.png ├── 应用场景.png ├── 编译1.png ├── 编译2.png ├── 初始化_2fp.png ├── 3fp第一次解析.png ├── 3fp第三次解析.png ├── 3fp第二次解析.png ├── 文件解析结果log.png ├── 新上线fp_3fp.png ├── 第一次解析_2fp.png └── 第二次解析_2fp.png ├── fileparsing ├── FileParsing_main.cpp └── src │ ├── recording_log.cpp │ ├── recording_log.h │ ├── file_parsing.h │ └── file_parsing.cpp ├── managerplatform ├── main.cpp ├── dialog.h ├── client_thread.h ├── dialog.cpp ├── client_thread.cpp └── dialog.ui ├── src ├── specifictime.h ├── client.h ├── server.h ├── client.cpp └── server.cpp ├── workstation ├── src │ ├── send_directory.h │ └── send_directory.cpp └── WorkStation_main.cpp ├── CMakeLists.txt ├── taskdistribution ├── src │ ├── distribution_server.h │ └── distribution_server.cpp └── TaskDistribution_main.cpp └── README.md /pictures/流程.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/流程.png -------------------------------------------------------------------------------- /pictures/应用场景.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/应用场景.png -------------------------------------------------------------------------------- /pictures/编译1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/编译1.png -------------------------------------------------------------------------------- /pictures/编译2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/编译2.png -------------------------------------------------------------------------------- /pictures/初始化_2fp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/初始化_2fp.png -------------------------------------------------------------------------------- /pictures/3fp第一次解析.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/3fp第一次解析.png -------------------------------------------------------------------------------- /pictures/3fp第三次解析.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/3fp第三次解析.png -------------------------------------------------------------------------------- /pictures/3fp第二次解析.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/3fp第二次解析.png -------------------------------------------------------------------------------- /pictures/文件解析结果log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/文件解析结果log.png -------------------------------------------------------------------------------- /pictures/新上线fp_3fp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/新上线fp_3fp.png -------------------------------------------------------------------------------- /pictures/第一次解析_2fp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/第一次解析_2fp.png -------------------------------------------------------------------------------- /pictures/第二次解析_2fp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/toMyLord/DistributedFileParsingSystem/HEAD/pictures/第二次解析_2fp.png -------------------------------------------------------------------------------- /fileparsing/FileParsing_main.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include 6 | #include "src/file_parsing.h" 7 | 8 | int main() { 9 | FileParsing file_parsing("127.0.0.1"); 10 | 11 | file_parsing.Init(); 12 | 13 | while (true) { 14 | file_parsing.Handler(); 15 | } 16 | } -------------------------------------------------------------------------------- /fileparsing/src/recording_log.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include "recording_log.h" 6 | 7 | RecordingLog * RecordingLog::log_ptr = nullptr; 8 | 9 | RecordingLog::RecordingLog() : out_file("./file_parse.log", ios::app){ 10 | 11 | } 12 | 13 | void RecordingLog::WriteLog(const string & log) { 14 | out_file << log << endl; 15 | } -------------------------------------------------------------------------------- /managerplatform/main.cpp: -------------------------------------------------------------------------------- 1 | #include "dialog.h" 2 | #include "client_thread.h" 3 | #include 4 | 5 | int main(int argc, char *argv[]) 6 | { 7 | QApplication a(argc, argv); 8 | Dialog w; 9 | ClientThread client_t("127.0.0.1", 7777); 10 | QObject::connect(&client_t, SIGNAL(Signal_t(QString)), &w, SLOT(ClientSlot(QString))); 11 | client_t.start(); 12 | w.show(); 13 | return a.exec(); 14 | } 15 | -------------------------------------------------------------------------------- /managerplatform/dialog.h: -------------------------------------------------------------------------------- 1 | #ifndef DIALOG_H 2 | #define DIALOG_H 3 | 4 | #include 5 | 6 | QT_BEGIN_NAMESPACE 7 | namespace Ui { class Dialog; } 8 | QT_END_NAMESPACE 9 | 10 | class Dialog : public QDialog 11 | { 12 | Q_OBJECT 13 | 14 | public: 15 | Dialog(QWidget *parent = nullptr); 16 | ~Dialog(); 17 | 18 | private: 19 | Ui::Dialog *ui; 20 | 21 | public slots: 22 | void ClientSlot(QString); 23 | }; 24 | #endif // DIALOG_H 25 | -------------------------------------------------------------------------------- /managerplatform/client_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef CLIENT_THREAD_H 2 | #define CLIENT_THREAD_H 3 | #include 4 | #include "../src/client.h" 5 | 6 | class ClientThread : public QThread 7 | { 8 | Q_OBJECT 9 | public: 10 | explicit ClientThread(char * ip, int port, QWidget *parent = nullptr); 11 | ~ClientThread() override; 12 | 13 | protected: 14 | void run() override; 15 | 16 | private: 17 | Client manager_client; 18 | int manager_fd; 19 | 20 | signals: 21 | void Signal_t(QString); 22 | 23 | public slots: 24 | }; 25 | 26 | #endif // CLIENT_THREAD_H 27 | -------------------------------------------------------------------------------- /src/specifictime.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/11/1. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_SPECIFICTIME_H 6 | #define DISTRIBUTEDFILEPARSING_SPECIFICTIME_H 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | 14 | class SpecificTime { 15 | public: 16 | string getTime(){ 17 | time_t now = time(0); 18 | tm * t = localtime(&now); 19 | 20 | 21 | stringstream time_stream; 22 | time_stream << t->tm_year + 1900 << "/" << t->tm_mon + 1 << "/" 23 | << t->tm_mday << " " << t->tm_hour << ":" << t->tm_min << ":" 24 | << t->tm_sec; 25 | 26 | return time_stream.str(); 27 | } 28 | }; 29 | 30 | 31 | #endif //DISTRIBUTEDFILEPARSING_SPECIFICTIME_H 32 | -------------------------------------------------------------------------------- /workstation/src/send_directory.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/11/1. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_SEND_DIRECTORY_H 6 | #define DISTRIBUTEDFILEPARSING_SEND_DIRECTORY_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "../../src/server.h" 14 | using namespace std; 15 | 16 | class SendDirectory { 17 | private: 18 | DIR * dp; 19 | struct dirent * dirp; 20 | string dir_path; 21 | public: 22 | SendDirectory() = default; 23 | 24 | bool setDir(string path); 25 | 26 | bool sendFile(Server * s, int fd, const char *dname); 27 | 28 | bool sendDir(Server * s, int fd); 29 | 30 | }; 31 | 32 | 33 | #endif //DISTRIBUTEDFILEPARSING_SEND_DIRECTORY_H 34 | -------------------------------------------------------------------------------- /managerplatform/dialog.cpp: -------------------------------------------------------------------------------- 1 | #include "dialog.h" 2 | #include "./ui_dialog.h" 3 | #include 4 | 5 | Dialog::Dialog(QWidget *parent) 6 | : QDialog(parent) 7 | , ui(new Ui::Dialog) 8 | { 9 | ui->setupUi(this); 10 | } 11 | 12 | Dialog::~Dialog() 13 | { 14 | delete ui; 15 | } 16 | 17 | void Dialog::ClientSlot(QString info) 18 | { 19 | QString flag = info.mid(0, 2); 20 | if(QString("ws") == flag) 21 | { 22 | ui->WorkstationInfo->setText(info.mid(3)); 23 | } 24 | else if(QString("fp") == flag) 25 | { 26 | ui->ParsingInfo->setText(info.mid(3)); 27 | } 28 | else 29 | { 30 | QByteArray sr=info.toLocal8Bit(); 31 | char ch[1024]; 32 | strcpy(ch,sr.data()); 33 | printf("cann't parsing server info:\t%s\n", ch); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /managerplatform/client_thread.cpp: -------------------------------------------------------------------------------- 1 | #include "client_thread.h" 2 | 3 | ClientThread::ClientThread(char * ip,int port, QWidget *parent) : manager_client(ip, port) 4 | { 5 | while(this->manager_client.Connect()) 6 | { 7 | qDebug("Waiting for task distribution server!\n"); 8 | sleep(2); 9 | } 10 | qDebug("connected to distribution server!\n"); 11 | } 12 | 13 | void ClientThread::run() 14 | { 15 | while(true) 16 | { 17 | char buffer[MAX_BUFFER_SIZE]; 18 | if(manager_client.Read(buffer) <= 0) 19 | { 20 | manager_client.Close(); 21 | break; 22 | } 23 | qDebug("READ:\t%s", buffer); 24 | QString info(buffer); 25 | emit Signal_t(info); 26 | } 27 | } 28 | 29 | ClientThread::~ClientThread() 30 | { 31 | this->manager_client.Close(); 32 | } 33 | -------------------------------------------------------------------------------- /fileparsing/src/recording_log.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_RECORDING_LOG_H 6 | #define DISTRIBUTEDFILEPARSING_RECORDING_LOG_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | using namespace std; 13 | 14 | //记录文件分析结果log的类，使用单例模式 15 | class RecordingLog { 16 | private: 17 | //阻止编译器自动生成copying操作 18 | RecordingLog(RecordingLog &) = delete; 19 | RecordingLog & operator=(const RecordingLog &) = delete; 20 | 21 | //必须为静态，类外会将其赋值nullptr 22 | static RecordingLog * log_ptr; 23 | 24 | ofstream out_file; 25 | 26 | public: 27 | RecordingLog(); 28 | 29 | //懒汉单例模式,设置为内联以提高效率 30 | static inline RecordingLog * getLogInstance() { 31 | if(log_ptr == nullptr){ 32 | log_ptr = new RecordingLog(); 33 | } 34 | return log_ptr; 35 | } 36 | 37 | void WriteLog(const string & log); 38 | }; 39 | 40 | 41 | #endif //DISTRIBUTEDFILEPARSING_RECORDING_LOG_H 42 | -------------------------------------------------------------------------------- /src/client.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_CLIENT_H 6 | #define DISTRIBUTEDFILEPARSING_CLIENT_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "specifictime.h" 18 | using namespace std; 19 | 20 | #ifndef MAX_BUFFER_SIZE 21 | #define MAX_BUFFER_SIZE 1024 22 | #endif 23 | 24 | class Client 25 | { 26 | private: 27 | int sock_fd; 28 | struct sockaddr_in client_addr; 29 | char server_ip[16]; 30 | int server_port; 31 | 32 | public: 33 | Client(char * server_ip, int server_port); 34 | 35 | int Connect(); 36 | 37 | char * getServerIP(); 38 | 39 | int getServerPort(); 40 | 41 | int Write(char * buff); 42 | 43 | int Read(char * buff); 44 | 45 | int Close(); 46 | 47 | virtual ~Client() = default; 48 | }; 49 | 50 | #endif //DISTRIBUTEDFILEPARSING_CLIENT_H 51 | -------------------------------------------------------------------------------- /src/server.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_SERVER_H 6 | #define DISTRIBUTEDFILEPARSING_SERVER_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "specifictime.h" 22 | using namespace std; 23 | 24 | #ifndef MAX_BUFFER_SIZE 25 | #define MAX_BUFFER_SIZE 1024 26 | #endif 27 | 28 | typedef struct { 29 | sockaddr_in client_sock; 30 | int client_fd; 31 | string client_ip; 32 | }ClientInfo; 33 | 34 | 35 | class Server { 36 | private: 37 | int listen_fd, listen_port, listen_size; 38 | sockaddr_in server_addr; 39 | public: 40 | Server(int port, int size); 41 | 42 | void Init(); 43 | 44 | void Listen(); 45 | 46 | int getListenFd(); 47 | 48 | int AcceptConnection(ClientInfo & client_info); 49 | 50 | int Write(int sock_fd, char buff[]); 51 | 52 | int Read(int sock_fd, char buff[]); 53 | 54 | int Close(int client_fd); 55 | 56 | virtual ~Server() = default; 57 | }; 58 | 59 | 60 | #endif //DISTRIBUTEDFILEPARSING_SERVER_H 61 | -------------------------------------------------------------------------------- /workstation/src/send_directory.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/11/1. 3 | // 4 | 5 | #include "send_directory.h" 6 | bool SendDirectory::setDir(string path) { 7 | if((dp = opendir(path.c_str())) == nullptr) { 8 | return false; 9 | } 10 | dir_path = path; 11 | return true; 12 | } 13 | 14 | bool SendDirectory::sendFile(Server * s, int fd, const char *dname) { 15 | char buffer[1024]; 16 | string file_name(dname); 17 | string file_path = dir_path + "/" + string(file_name); 18 | 19 | ifstream in_file(file_path.c_str()); 20 | in_file.is_open(); 21 | 22 | in_file.seekg(0, in_file.end); 23 | int file_size = in_file.tellg(); 24 | in_file.seekg(0, in_file.beg); 25 | 26 | in_file.read(buffer, 8); 27 | buffer[8] = '\0'; 28 | 29 | stringstream send_buffer; 30 | send_buffer << file_name << ":" << buffer << ";" << file_size; 31 | 32 | strcpy(buffer, send_buffer.str().c_str()); 33 | 34 | s->Write(fd, buffer); 35 | } 36 | 37 | bool SendDirectory::sendDir(Server * s, int fd) { 38 | while((dirp = readdir(dp)) != nullptr) { 39 | if(strncmp(dirp->d_name, ".", 1) == 0 || strncmp(dirp->d_name, "..", 2) == 0) 40 | continue; 41 | 42 | sendFile(s, fd, dirp->d_name); 43 | } 44 | s->Write(fd, "SendDirFinished"); 45 | closedir(dp); 46 | } 47 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(DistributedFileParsingSystem) 3 | 4 | set(CMAKE_CXX_STANDARD 11) 5 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 6 | 7 | find_package(Qt5 COMPONENTS Widgets REQUIRED) 8 | # 开启Qt代码自动生成，不再需用自己手写了。按顺序，分别是`Q_OBJECT`宏展开，资源文件，界面文件。 9 | set(CMAKE_AUTOUIC ON) 10 | set(CMAKE_AUTOMOC ON) 11 | set(CMAKE_AUTORCC ON) 12 | 13 | set(CMKAE_THREAD_PREFER_PTHREAD) 14 | 15 | 16 | aux_source_directory(fileparsing/src FileParsing_src) 17 | aux_source_directory(taskdistribution/src TaskDistribution_src) 18 | aux_source_directory(workstation/src WorkStation_src) 19 | aux_source_directory(managerplatform ManagerPlatform_src) 20 | aux_source_directory(src CS_src) 21 | 22 | 23 | add_executable(FileParsing 24 | ./fileparsing/FileParsing_main.cpp 25 | ${FileParsing_src} 26 | ${CS_src}) 27 | add_executable(TaskDistribution 28 | ./taskdistribution/TaskDistribution_main.cpp 29 | ${TaskDistribution_src} 30 | ${CS_src}) 31 | add_executable(WorkStation 32 | ./workstation/WorkStation_main.cpp 33 | ${WorkStation_src} 34 | ${CS_src}) 35 | add_executable(ManagerPlatform 36 | ${ManagerPlatform_src} 37 | ${CS_src} 38 | ) 39 | 40 | target_link_libraries(TaskDistribution pthread) 41 | target_link_libraries(WorkStation pthread) 42 | target_link_libraries(ManagerPlatform PRIVATE Qt5::Widgets) -------------------------------------------------------------------------------- /taskdistribution/src/distribution_server.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_DISTRIBUTION_SERVER_H 6 | #define DISTRIBUTEDFILEPARSING_DISTRIBUTION_SERVER_H 7 | 8 | #include "../../src/server.h" 9 | #include 10 | #include 11 | 12 | enum ClientState {WAITING, TRANSMITING}; 13 | 14 | typedef struct { 15 | ClientInfo client_info; 16 | ClientState client_state; 17 | }ClientNode; 18 | 19 | 20 | class DistributionServer { 21 | private: 22 | Server server; //选择聚合的方式，因为这里是“根据某物实现出(is-implemented-in-terms-of)”的理念 23 | 24 | //用来存储所有连接至任务分配服务器的工作站 25 | vector client_node; 26 | 27 | bool isthreaded; 28 | 29 | public: 30 | DistributionServer(int port, int size); 31 | 32 | int getListenFd(); 33 | 34 | ClientNode getClientNode(int fd) const; 35 | 36 | int AcceptConnection(ClientNode & client); //增加了心跳机制 37 | 38 | void setState(int sock_fd, ClientState state); 39 | 40 | int Write(int sock_fd, char buff[]); 41 | 42 | int Read(int sock_fd, char buff[]); 43 | 44 | int Close(int sock_fd); 45 | 46 | vector getClientNode() const; 47 | 48 | void getAllClientInfo(string & buff); 49 | 50 | void SendHeartbeats(DistributionServer * server); 51 | 52 | static void Heartbeats_t(DistributionServer * server); 53 | 54 | }; 55 | 56 | #endif //DISTRIBUTEDFILEPARSING_DISTRIBUTION_SERVER_H 57 | -------------------------------------------------------------------------------- /fileparsing/src/file_parsing.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #ifndef DISTRIBUTEDFILEPARSING_FILE_PARSING_H 6 | #define DISTRIBUTEDFILEPARSING_FILE_PARSING_H 7 | 8 | #include "../../src/client.h" 9 | #include "recording_log.h" 10 | 11 | class FileParsing; 12 | 13 | //状态模式的基类 14 | class State { 15 | public: 16 | virtual void Handler(FileParsing * fp) = 0; 17 | //析构函数设置成虚函数避免派生类在析构的时候只调用基类的析构函数造成不完全析构 18 | virtual ~State() = default; 19 | }; 20 | 21 | class WatingState : public State { 22 | public: 23 | void Handler(FileParsing * fp); 24 | 25 | ~WatingState() = default; 26 | }; 27 | 28 | //文件解析状态,在这种状态下会使用记录log的类RecordingLog 29 | class ParsingState : public State { 30 | public: 31 | void Handler(FileParsing * fp); 32 | 33 | ~ParsingState() = default; 34 | }; 35 | 36 | 37 | //文件解析类，使用状态模式，在解析文件状态和等待连接状态中切换 38 | class FileParsing { 39 | private: 40 | //用来接收派生类的指针 41 | State * state; 42 | Client parsing_client; 43 | 44 | WatingState * ws; 45 | ParsingState * ps; 46 | 47 | public: 48 | char workstation_ip[16]; 49 | int workstation_port; 50 | 51 | FileParsing(char * ip, int port = 6666); 52 | 53 | void Init(); 54 | 55 | inline void setState(State * s); 56 | 57 | void Handler(); 58 | 59 | WatingState * getWatingState(); 60 | 61 | ParsingState * getParsingState(); 62 | 63 | Client & getClient(); 64 | 65 | ~FileParsing(); 66 | }; 67 | #endif //DISTRIBUTEDFILEPARSING_FILE_PARSING_H 68 | -------------------------------------------------------------------------------- /managerplatform/dialog.ui: -------------------------------------------------------------------------------- 1 | 2 | 3 | Dialog 4 | 5 | 6 | 7 | 0 8 | 0 9 | 582 10 | 282 11 | 12 | 13 | 14 | Dialog 15 | 16 | 17 | 18 | 19 | 20 20 | 51 21 | 256 22 | 201 23 | 24 | 25 | 26 | 27 | 28 | 29 | 20 30 | 10 31 | 181 32 | 31 33 | 34 | 35 | 36 | 文件解析服务器: 37 | 38 | 39 | 40 | 41 | 42 | 300 43 | 10 44 | 101 45 | 31 46 | 47 | 48 | 49 | 工作站: 50 | 51 | 52 | 53 | 54 | 55 | 300 56 | 51 57 | 251 58 | 201 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /src/client.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include "client.h" 6 | 7 | Client::Client(char * server_ip, int server_port) 8 | { 9 | strncpy(this->server_ip, server_ip, 16); 10 | this->server_port = server_port; 11 | 12 | struct hostent * host; 13 | if((host = gethostbyname(this->server_ip)) == NULL) 14 | { 15 | SpecificTime st; 16 | fprintf(stderr, "[%s Host Error]:\tThe host name %s is illegal.\n", st.getTime().c_str(), server_ip); 17 | exit(1); 18 | } 19 | 20 | if((this->sock_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) 21 | { 22 | SpecificTime st; 23 | fprintf(stderr, "[%s Socket Error]:\tInit socket fd error!\n", st.getTime().c_str()); 24 | exit(1); 25 | } 26 | 27 | bzero(&this->client_addr, sizeof(this->client_addr)); 28 | this->client_addr.sin_family = AF_INET; 29 | this->client_addr.sin_port = htons(this->server_port); 30 | this->client_addr.sin_addr = *((struct in_addr *)host->h_addr); 31 | 32 | } 33 | 34 | 35 | int Client::Connect() 36 | { 37 | int nbytes = connect(this->sock_fd, (struct sockaddr *)(&this->client_addr), 38 | sizeof(struct sockaddr)); 39 | 40 | return nbytes; 41 | } 42 | 43 | char * Client::getServerIP() { 44 | return this->server_ip; 45 | } 46 | 47 | int Client::getServerPort() { 48 | return this->server_port; 49 | } 50 | 51 | int Client::Write(char * buff) 52 | { 53 | int nbytes = 0; 54 | if ((nbytes = write(this->sock_fd, buff, strlen(buff))) == -1) { 55 | SpecificTime st; 56 | fprintf(stderr, "[%s Write Error]:\t%s\n", st.getTime().c_str(), strerror(errno)); 57 | } 58 | 59 | return nbytes; 60 | } 61 | 62 | int Client::Read(char * buff) 63 | { 64 | int nbytes = 0; 65 | if ((nbytes = read(this->sock_fd, buff, MAX_BUFFER_SIZE)) == -1) { 66 | SpecificTime st; 67 | fprintf(stderr, "[%s Read Error]:\t%s\n", st.getTime().c_str(), strerror(errno)); 68 | } else 69 | buff[nbytes] = '\0'; 70 | 71 | return nbytes; 72 | } 73 | 74 | int Client::Close() { 75 | close(this->sock_fd); 76 | } -------------------------------------------------------------------------------- /src/server.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include "server.h" 6 | 7 | Server::Server(int port, int size): listen_port(port), listen_size(size) { 8 | this->Init(); 9 | this->Listen(); 10 | } 11 | 12 | void Server::Init() { 13 | if((listen_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) 14 | { //服务器端开始建立socket描述符 15 | SpecificTime st; 16 | fprintf(stderr, "[%s Socket Error]:\t%s \n\a", st.getTime().c_str(), strerror(errno)); 17 | exit(1); 18 | } 19 | //服务器端填充tcp sockaddr结构 20 | bzero(&server_addr, sizeof(struct sockaddr_in)); //先将套接字地址数据结构清零 21 | server_addr.sin_family = AF_INET; 22 | server_addr.sin_addr.s_addr = htons(INADDR_ANY); 23 | server_addr.sin_port = htons(listen_port); 24 | 25 | if(bind(listen_fd, (struct sockaddr *)(&server_addr), sizeof(struct sockaddr)) == -1) 26 | { 27 | SpecificTime st; 28 | fprintf(stderr, "[%s Bind Error]:\t%s\n\a", st.getTime().c_str(), strerror(errno)); 29 | exit(1); 30 | } 31 | } 32 | 33 | void Server::Listen() { 34 | if(listen(listen_fd, listen_size) == -1) 35 | { //端口绑定成功，监听socketfd描述符，同时处理的最大连接请求数为10 36 | SpecificTime st; 37 | fprintf(stderr, "[%s Listen Error]:\t%s\n\a", st.getTime().c_str(), strerror(errno)); 38 | exit(1); 39 | } 40 | } 41 | 42 | int Server::getListenFd() { 43 | return this->listen_fd; 44 | } 45 | 46 | int Server::AcceptConnection(ClientInfo & client_info) { 47 | int sockaddr_size = sizeof(struct sockaddr_in); 48 | 49 | if ((client_info.client_fd = accept(listen_fd, (struct sockaddr *)(&client_info.client_sock), 50 | (socklen_t *) &sockaddr_size)) == -1) 51 | { //调用accept接受一个连接请求 52 | SpecificTime st; 53 | fprintf(stderr, "[%s Accept error]:\t%s\n\a", st.getTime().c_str(), strerror(errno)); 54 | exit(1); 55 | } 56 | 57 | client_info.client_ip.assign(inet_ntoa(client_info.client_sock.sin_addr)); 58 | 59 | return client_info.client_fd; 60 | } 61 | 62 | int Server::Write(int sock_fd, char buff[]) { 63 | int nbytes = 0; 64 | if((nbytes = write(sock_fd, buff, strlen(buff))) == -1) { 65 | SpecificTime st; 66 | fprintf(stderr, "[%s Write Error]:\t%s\n", st.getTime().c_str(), strerror(errno)); 67 | } 68 | return nbytes; 69 | } 70 | 71 | int Server::Read(int sock_fd, char buff[]) { 72 | int nbytes = 0; 73 | if ((nbytes = read(sock_fd, buff, MAX_BUFFER_SIZE)) == -1) { 74 | SpecificTime st; 75 | fprintf(stderr, "[%s Read Error]:\t%s\n", st.getTime().c_str(), strerror(errno)); 76 | } else 77 | buff[nbytes] = '\0'; 78 | 79 | return nbytes; 80 | } 81 | 82 | int Server::Close(int client_fd) { 83 | close(client_fd); 84 | } -------------------------------------------------------------------------------- /taskdistribution/src/distribution_server.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | #include "distribution_server.h" 5 | 6 | DistributionServer::DistributionServer(int port, int size) : server(port, size), client_node(), isthreaded(false) { 7 | this->server.Listen(); 8 | } 9 | 10 | 11 | int DistributionServer::getListenFd() { 12 | return this->server.getListenFd(); 13 | } 14 | 15 | ClientNode DistributionServer::getClientNode(int fd) const { 16 | auto client_it = find_if(client_node.begin(), client_node.end(), 17 | [fd](const ClientNode &cli){ return cli.client_info.client_fd == fd; }); 18 | return *client_it; 19 | } 20 | 21 | int DistributionServer::AcceptConnection(ClientNode & client) { 22 | int fd = server.AcceptConnection(client.client_info); 23 | 24 | client.client_state = WAITING; 25 | 26 | client_node.push_back(client); 27 | 28 | if(isthreaded == false) { 29 | isthreaded = true; 30 | 31 | thread heartbeats_t(this->Heartbeats_t, this); 32 | heartbeats_t.detach(); 33 | } 34 | 35 | return fd; 36 | } 37 | 38 | void DistributionServer::setState(int sock_fd, ClientState state) { 39 | auto client_it = find_if(client_node.begin(), client_node.end(), 40 | [sock_fd](const ClientNode &cli){ return cli.client_info.client_fd == sock_fd; }); 41 | client_it->client_state = state; 42 | } 43 | 44 | int DistributionServer::Write(int sock_fd, char buff[]) { 45 | return server.Write(sock_fd, buff); 46 | } 47 | 48 | int DistributionServer::Read(int sock_fd, char buff[]) { 49 | return server.Read(sock_fd, buff); 50 | } 51 | 52 | int DistributionServer::Close(int sock_fd) { 53 | auto client_it = find_if(client_node.begin(), client_node.end(), 54 | [sock_fd](const ClientNode &cli){ return cli.client_info.client_fd == sock_fd; }); 55 | 56 | client_node.erase(client_it); 57 | 58 | close(sock_fd); 59 | } 60 | 61 | void DistributionServer::getAllClientInfo(string & buff) { 62 | for(auto a : client_node) { 63 | buff += a.client_info.client_ip; 64 | buff += '\t'; 65 | buff += a.client_state == WAITING ? "WAITING" : "TRANSMITING"; 66 | buff += '\n'; 67 | } 68 | } 69 | 70 | vector DistributionServer::getClientNode() const { 71 | return this->client_node; 72 | } 73 | 74 | void DistributionServer::SendHeartbeats(DistributionServer * server) { 75 | //如果有连接，就每隔5秒发送一次心跳包。 76 | while(true) { 77 | if (!server->getClientNode().empty()) 78 | for (auto a : server->getClientNode()) 79 | server->Write(a.client_info.client_fd, "HeartBeats"); 80 | 81 | sleep(5); 82 | } 83 | } 84 | 85 | void DistributionServer::Heartbeats_t(DistributionServer * server) { 86 | server->SendHeartbeats(server); 87 | } -------------------------------------------------------------------------------- /workstation/WorkStation_main.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include "../src/server.h" 6 | #include "../src/client.h" 7 | #include "src/send_directory.h" 8 | #include 9 | 10 | Client distributer_client("127.0.0.1", 5555); 11 | SpecificTime st; 12 | 13 | 14 | void HeartBeatsHandler_t(); 15 | 16 | int main() { 17 | Server parsing_server(8888, 1); 18 | 19 | while(distributer_client.Connect() != 0) 20 | sleep(1); 21 | 22 | thread heartbeats_t(HeartBeatsHandler_t); 23 | cout << "[" << st.getTime().c_str() << " Workstation Connected]:\tWorkstation connected to distribution server!" << endl; 24 | 25 | heartbeats_t.detach(); 26 | 27 | parsing_server.Listen(); 28 | 29 | cout << "[" << st.getTime().c_str() << " Start Listening]:\tWorkstation server is listening,waiting for file parser connect!" << endl; 30 | 31 | while(true) { 32 | ClientInfo parsing_info; 33 | SendDirectory sd; 34 | 35 | //获取发文件的前提信息 36 | cout << "Please enter the directory you want to send:\t"; 37 | string directory_path; 38 | while(true) { 39 | cin >> directory_path; 40 | if(sd.setDir(directory_path) == true) 41 | break; 42 | else 43 | cout << "Directory path is illegal, please enter again:\t"; 44 | } 45 | 46 | distributer_client.Write("ParsingRequest"); 47 | int fd = parsing_server.AcceptConnection(parsing_info); 48 | 49 | cout << "[" << st.getTime().c_str() << " Start Sending]:\tFile parser is connected and server started sending!" << endl; 50 | 51 | //文件发送过程 52 | sd.sendDir(&parsing_server, fd); 53 | 54 | distributer_client.Write("TransmitSuccess"); 55 | 56 | parsing_server.Close(fd); 57 | cout << "[" << st.getTime().c_str() << " Finished Sending]:\tFinished sending and close workstation server" << endl; 58 | } 59 | } 60 | 61 | void HeartBeatsHandler_t() { 62 | char buffer[MAX_BUFFER_SIZE]; 63 | 64 | while(true) { 65 | if(distributer_client.Read(buffer) == 0) { 66 | distributer_client.Close(); 67 | cout << "[" << st.getTime().c_str() << " Distributer Closed]:\tDisconnected from distributer server" << endl; 68 | exit(0); 69 | } 70 | if (strncmp(buffer, "HeartBeats", 10) == 0) { 71 | //收到心跳包 72 | // cout << "[" << st.getTime().c_str() << 73 | // " Heart Beats]:\tReceived heart beats packet from distribution server!" << endl; 74 | 75 | strcpy(buffer, "HeartBeats"); 76 | 77 | distributer_client.Write(buffer); 78 | } else { 79 | //无法解析接收内容 80 | cout << "[" << st.getTime().c_str() << 81 | " Reply Error]:\tWorkstation server cann't parse the reply information from distribution server : " << 82 | buffer << endl; 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /fileparsing/src/file_parsing.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | 5 | #include "file_parsing.h" 6 | 7 | FileParsing::FileParsing(char * ip, int port) : parsing_client(ip, port){ 8 | ws = new WatingState(); 9 | ps = new ParsingState(); 10 | } 11 | 12 | void FileParsing::Init() { 13 | SpecificTime st; 14 | while (parsing_client.Connect() == -1) { 15 | sleep(1); 16 | } 17 | 18 | cout << "[" << st.getTime().c_str() << " Connected]:\tconnected to task distributer server at port 6666" << endl; 19 | 20 | setState(ws); 21 | } 22 | 23 | inline void FileParsing::setState(State * s) { 24 | state = s; 25 | } 26 | 27 | void FileParsing::Handler() { 28 | state->Handler(this); 29 | } 30 | 31 | WatingState * FileParsing::getWatingState() { 32 | return this->ws; 33 | } 34 | 35 | ParsingState * FileParsing::getParsingState() { 36 | return this->ps; 37 | } 38 | 39 | Client & FileParsing::getClient() { 40 | return this->parsing_client; 41 | } 42 | 43 | FileParsing::~FileParsing() { 44 | delete ws; 45 | delete ps; 46 | 47 | parsing_client.Close(); 48 | } 49 | 50 | void WatingState::Handler(FileParsing * fp) { 51 | char buffer[MAX_BUFFER_SIZE]; 52 | 53 | SpecificTime st; 54 | cout << "[" << st.getTime().c_str() << " Waiting Tasks]:\tWating for tasks from task distributer server!" << endl; 55 | 56 | while(true) { 57 | if (fp->getClient().Read(buffer) <= 0) { 58 | fp->getClient().Close(); 59 | return; 60 | } 61 | if (strncmp(buffer, "HeartBeats", 10) == 0) { 62 | //收到心跳包 63 | // cout << "[" << st.getTime().c_str() << 64 | // " Heart Beats]:\tReceived heart beats packet from distribution server!" << endl; 65 | 66 | strcpy(buffer, "HeartBeats"); 67 | fp->getClient().Write(buffer); 68 | } else { 69 | break; 70 | } 71 | } 72 | strncpy(fp->workstation_ip, buffer, 16); 73 | fp->workstation_port = 8888; 74 | 75 | fp->setState(fp->getParsingState()); 76 | } 77 | 78 | 79 | 80 | void ParsingState::Handler(FileParsing * fp) { 81 | char buffer[1024]; 82 | Client workstation_client(fp->workstation_ip, fp->workstation_port); 83 | 84 | workstation_client.Connect(); 85 | 86 | SpecificTime st; 87 | cout << "[" << st.getTime().c_str() << " Started Parsing]:\tConnected to workstation and start parsing!" << endl; 88 | 89 | //对文件进行解析 90 | while(true) { 91 | if(workstation_client.Read(buffer) <= 0) { 92 | cout <<"read0!"<getClient().Write(buff); 95 | 96 | cout << "[" << st.getTime().c_str() << " Finished Parsing]:\tFinished parsing and reported to task distributer server!" << endl; 97 | 98 | fp->setState(fp->getWatingState()); 99 | return; 100 | } 101 | if(strncmp(buffer, "SendDirFinished", 15) == 0) 102 | break; 103 | 104 | string buff(buffer); 105 | int slice_l = buff.find(':'); 106 | int slice_r = buff.find(';'); 107 | 108 | string file_name = buff.substr(0, slice_l); 109 | string file_8 = buff.substr(slice_l + 1, 8); 110 | int file_lenth = stoi(buff.substr(slice_r + 1)); 111 | 112 | stringstream log; 113 | log << "[" << st.getTime().c_str() << " " << fp->getClient().getServerIP() 114 | << " " << fp->getClient().getServerPort() << "]:\t" << file_name 115 | << "\t" << file_8 << "\t" << file_lenth << "bytes." << endl; 116 | 117 | cout << log.str(); 118 | 119 | RecordingLog::getLogInstance()->WriteLog(log.str()); 120 | } 121 | 122 | char buff[] = "ParsingSuccess"; 123 | fp->getClient().Write(buff); 124 | 125 | cout << "[" << st.getTime().c_str() << " Finished Parsing]:\tFinished parsing and reported to task distributer server!" << endl; 126 | 127 | fp->setState(fp->getWatingState()); 128 | } 129 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # C++实现分布式文件解析系统 2 | 3 | ## 一、该系统的应用场景 4 | 5 | 整个分布式文件解析系统一共有四个角色：任务分配服务器、文件解析服务器、工作站和管理平台。在一个大型的网络中，有若干个用户（即“工作站”）随机地通过“任务分配服务器”向若干个“文件解析服务器”提交文本文件。“文件解析服务器”将收到的文件解析后写入log文件中。“管理平台”负责以可视化的界面观察所有连接到的“任务分配服务器”和“工作站”的ip及工作状态。 6 | 7 | 8 | 9 | ![应用场景](pictures/应用场景.png) 10 | 11 | ### 1、任务分配服务器 12 | 13 | 任务分配服务器的作用是轮流地将来自“工作站”的文件传送请求转发到每个在线的“文件解析服务器”上，从而避免某个特定的“文件解析服务器”工作负载太重。“任务分配服务器”轮流给所有在线的“文件解析服务器”分配任务。轮流的原则是按每工作站每批轮流，不考虑每批文件的数量多少。当有“工作站”准备传送一批文件时，首先连接任务分配服务器，通知任务分配服务器有一批文件待传。任务分配服务器则把该工作站的IP和端口告诉选定的“文件解析服务器”，以便让“文件分析服务器”能主动和“工作站”连接，并完成该批文件传输。 14 | 15 | ps：主动的含义是文件分析服务器主动通过`connect`连接工作站。“任务分配服务器”本身不接收或转发文件。工作站每次都是按批提交文件，一批文件的数量不等。 16 | 17 | ### 2、文件解析服务器 18 | 19 | “文件解析服务器”每收到用户的一批文件后，就自动对该批文件进行“文件解析”。“文件解析”的功能是： 20 | 21 | + 1、返回文件的前面8个字节和文件长度。 22 | + 2、把功能1的结果保存起来，具体就是把“文件名，来源IP，来源端口，处理日期，处理时间，前面8个字节，文件长度”作为一条记录写到本地日志文件中，一条记录就是一行。 23 | 24 | ### 3、管理平台 25 | 26 | 管理平台的作用是采用图形化的方式展示目前在线的工作站，文件分析服务器的工作状态。能展示工作站的IP，工作状态（传文件中或没有传文件）。能展示文件分析服务器的IP，工作状态（接收文件中，文件分析中，没有传文件，没有文件被分析）。当一个工作站或文件分析服务器宕机，能实时更新状态。 27 | 28 | ## 二、系统实现 29 | 30 | 整个系统的工作流程如下图所示： 31 | 32 | ![流程](pictures/流程.png) 33 | 34 | 该项目源码在 35 | 36 | ## 1、DistributedFileParsingSystem/src 37 | 38 | src目录下包含`Server`类、`Client`类、`SpecificTime`类。`Server`类用于服务器初始化，监听`socket_fd`，接受来自客户端的连接请求，接收或发送信息，关闭套接字。`Client`类用于客户端初始化，连接服务器，与服务器进行全双工通信，关闭连接。`SpecificTime`类用于获取当前系统的时间，并以规定的格式返回一个`std::string`类型的字符串。上述三个类会被用于所有角色，因此以一个目录的形式单独放在所有角色所属目录外，通过CmakeLists.txt将这些文件的编译相关性联系在一起，从而生成可执行文件。 39 | 40 | ## 2、DistributedFileParsingSystem/taskdistribution 41 | 42 | 该目录下为编译`TaskDistribution`所需要的专属文件。在当前目录下的src目录内，定义了`DistributionServer`类，该类内组合了DistributedFileParsingSystem/src目录下的`Server`类。实现`DistributionServer`类的目的是为了使用该类监听特定端口的所有网络连接，并将所有连接的客户端信息都存储在一个`std::vector`容器内。`DistributionServer`类的定义如下： 43 | 44 | ```C++ 45 | enum ClientState {WAITING, TRANSIMITING}; 46 | 47 | typedef struct { 48 | ClientInfo client_info; 49 | ClientState client_state; 50 | }ClientNode; 51 | 52 | class DistributionServer { 53 | private: 54 | Server server; 55 | 56 | //用来存储所有连接至任务分配服务器的工作站 57 | vector client_node; 58 | 59 | public: 60 | ...... 61 | }; 62 | ``` 63 | 64 | 在`ClientNode`结构体内嵌套的`ClientInfo`结构体声明和定义在DistributedFileParsingSystem/src/server.h文件内，`ClientInfo`结构体保存了连接到`server`的客户端的`sockaddr_in`、`socket_fd`以及ip信息。`ClientNode`结构体扩展了一个新的字段`client_state`，该字段用来描述客户端的传输状态。 65 | 66 | 通过`std::vector`将所有连接的客户端信息都保存下来。每当有新的客户端连接，就使用`client_node.push_back()`将信息保存在内存中。如果有已连接的客户端断开连接，需要用`find_if()`函数配合`lambda`表达式找到在`vector`中的迭代器，然后断开连接并删除`client_node`中对应的数据。具体过程如下： 67 | 68 | ```C++ 69 | auto client_it = find_if(client_node.begin(), client_node.end(), 70 | [sock_fd](const ClientNode &cli){ return cli.client_info.client_fd == sock_fd; }); 71 | 72 | client_node.erase(client_it); 73 | 74 | close(sock_fd); 75 | ``` 76 | 77 | TaskDistribution_main.cpp是任务分配服务器可执行文件编译所需的主函数。在该文件内声明了两个`DistributionServer`类的实例`workstation_server`和`parsing_server`，一个`Server`类的实例`manager_server`，分别用来处理来自工作站、文件解析服务器、管理平台的连接。因为任务分配服务器只允许一个管理平台连接，因此管理平台选择`Server`类，因为可能会有大量工作站和文件解析服务器连接至任务分配服务器，因此选择`DistributionServer`来分别监听工作站和文件解析服务器的连接端口。 78 | 79 | 在主函数中，我选择使用`epoll`——IO多路复用技术来监听工作站、文件解析服务器和管理平台的连接请求，从而实现在单进程中对三个端口进行监听。 80 | 81 | 当有管理平台连接至任务分配服务器后，任务分配服务器就不再处理来自管理平台的连接请求，直到当前管理平台退出。 82 | 83 | 当有文件解析服务器连接至任务分配服务器后，任务分配服务器会将文件解析添加至就绪队列，等待来自工作站的请求。每当有工作站发出文件传输请求，就将处于就绪队列顶端的文件解析服务器出队列，并分配给当前工作站用来处理解析任务。处理完文件解析任务后，任务分配服务器会将文件解析服务器重新加入队列。 84 | 85 | 当工作站连接至任务分配服务器后，任务分配服务器会建立一个新的线程用来处理工作站发出的请求。当工作站发出文件解析请求后，该线程会将就绪队列顶端的文件解析服务器分配给工作站，并一直监控整个解析过程，直到解析完成，将文件解析服务器重新入队列。完成后此线程会重新监听来自工作站的解析请求，直到工作站断开连接，该线程就会消亡。如果工作站发出解析请求后，没有任务文件解析服务器就绪，那么任务分配服务器会每过1s检测一次是否有就绪的文件解析服务器（此处可以改进，将每秒轮询的方式改为等待时长者优先策略）。 86 | 87 | **注意**：新的任务解析服务器加入了心跳机制，会主动向所有连接的工作站或文件解析服务器主动的发送心跳包，有效的解决了客户端断开连接或者宕机的情况。同时，取消了申请单个线程对新的解析请求进行跟踪。转而使用统一的方式与其他客户端进行交互，有效提高了文件解析服务器的可靠性。 88 | 89 | ## 3、DistributedFileParsingSystem/fileparsing 90 | 91 | 该目录下为编译`TaskDistribution`所需要的专属文件。在当前目录下的src目录内，主要实现两个功能：**使用单例模式实现日志文件记录**、**使用状态模式实现文件解析服务器在等待状态和解析状态之间的切换**。 92 | 93 | 使用单例模式来实现日志记录的`RecordingLog`类的实例只会创建一次，从而避免创建多个日志类实例，造成写入混乱的情况。`RecordingLog`类会将特定信息以`filesteam`的方式写入文件中，具体被记录在日志文件中的信息包括接收来自工作站文件的时间、日期、工作站ip、工作站端口、文件名、文件前8个字节内容、文件总长度。该类的声明情况如下所示： 94 | 95 | ```C++ 96 | //记录文件分析结果log的类，使用单例模式 97 | class RecordingLog { 98 | private: 99 | //阻止编译器自动生成copying操作 100 | RecordingLog(RecordingLog &) = delete; 101 | RecordingLog & operator=(const RecordingLog &) = delete; 102 | 103 | //必须为静态，类外会将其赋值nullptr 104 | static RecordingLog * log_ptr; 105 | 106 | ofstream out_file; 107 | 108 | public: 109 | RecordingLog(); 110 | 111 | //懒汉单例模式在类内实现的函数会被编译器自动转化为内联函数。使用内联函数的目的是在调用时提高效率。 112 | static inline RecordingLog * getLogInstance() { 113 | if(log_ptr == nullptr){ 114 | log_ptr = new RecordingLog(); 115 | } 116 | return log_ptr; 117 | } 118 | 119 | void WriteLog(const string & log); 120 | }; 121 | 122 | RecordingLog * RecordingLog::log_ptr = nullptr; 123 | ``` 124 | 125 | 在本系统的FileParsing程序中，使用状态模式用来处理文件解析服务器的状态转换。文件解析服务器主要有两种状态——等待任务分配服务器发送解析文件请求状态和接受来自工作站的文件并完成解析，写入日志文件的状态。 126 | 127 | 在第一种状态下，文件解析服务器收到来自任务分配服务器的解析请求后，会将需要主动连接的工作站的ip及端口保存下来，并切换到第二种状态。 128 | 129 | 在第二种状态下，文件解析服务器根据保存的ip和端口主动连接工作站，连接完成后接收来自工作站的文件信息，解析后通过`RecordingLog`类中的`getLogInstance()`函数将解析后的结果写入日志文件，完成后会将解析完成情况回传给任务分配服务器，在任务分配服务器中，将该文件解析服务器重新添加到就绪队列。之后会主动切换到第一种状态，再次等待任务请求。 130 | 131 | ```C++ 132 | class FileParsing; 133 | 134 | //状态模式的基类 135 | 136 | class State { 137 | public: 138 | virtual void Handler(FileParsing * fp) = 0; 139 | //析构函数设置成虚函数避免派生类在析构的时候只调用基类的析构函数造成不完全析构 140 | virtual ~State() = default; 141 | }; 142 | 143 | //等待连接状态 144 | class WatingState : public State { 145 | public: 146 | void Handler(FileParsing * fp); 147 | 148 | ~WatingState() = default; 149 | }; 150 | 151 | //文件解析状态,在这种状态下会使用记录log的类RecordingLog 152 | class ParsingState : public State { 153 | public: 154 | void Handler(FileParsing * fp); 155 | 156 | ~ParsingState() = default; 157 | }; 158 | 159 | 160 | //文件解析类，使用状态模式，在解析文件状态和等待连接状态中切换 161 | class FileParsing { 162 | private: 163 | //用来接收派生类的指针 164 | State * state; 165 | Client parsing_client; 166 | 167 | WatingState * ws; 168 | ParsingState * ps; 169 | 170 | public: 171 | char workstation_ip[16]; 172 | int workstation_port; 173 | 174 | FileParsing(char * ip, int port = 6666); 175 | 176 | void Init(); 177 | 178 | inline void setState(State * s); 179 | 180 | void Handler(); 181 | 182 | WatingState * getWatingState(); 183 | 184 | ParsingState * getParsingState(); 185 | 186 | ~FileParsing(); 187 | }; 188 | 189 | 190 | void FileParsing::Handler() { 191 | state->Handler(this); 192 | } 193 | 194 | 195 | 196 | void WatingState::Handler(FileParsing * fp) { 197 | ...... 198 | 199 | fp->setState(fp->getParsingState()); 200 | } 201 | 202 | 203 | 204 | void ParsingState::Handler(FileParsing * fp) { 205 | ....... 206 | fp->setState(fp->getWatingState()); 207 | } 208 | ``` 209 | 210 | 这部分程序使用了状态模式，使得文件解析服务器在等待分配任务状态及解析文件状态自动切换，从而保证了两种状态上的逻辑分离，具有非常好的封装性，并体现了开闭原则和单一职责原则：每个状态都是一个子类，如果需要增加状态就只需要增加子类，如果需要修改状态，就只需要修改一个子类即可完成，具有很高的扩展性。 211 | 212 | ## 4、DistributedFileParsingSystem/workstation 213 | 214 | 该目录下为编译`WorkStation`所需要的专属文件。在当前目录下的src目录内，定义了`SendDirectory`类。通过该类，可以实现根据输入的一个目录，自动发送该目录下的所有文件。注意，该目录下不可再包含目录。 215 | 216 | ## 5、DistributedFileParsingSystem/managerplatform 217 | 218 | 该目录下为编译`Managerplatform`所需要的专属文件。此目录可以使用QtCreater打开，但是编译还是需要根据DistributedFileParsingSystem/CMakeLists.txt文件通过cmake编译。 219 | 220 | 该目录下的`ClientThread`类通过继承`QThread`类，实现在Qt中使用线程。在该部分程序运行时，就会创建一个`ClientThread`线程，用于与任务分配服务器建立连接，并不断接受来自任务分配服务发送的有关工作站及文件解析服务器的在线状态以及传输状态。每当有新的信息后，就通过Qt的槽机制，发送给`QDialog`，显示在界面中。 221 | 222 | ## 三、编译及运行 223 | 224 | ## 1、编译过程 225 | 226 | ```shell 227 | cd ~ 228 | git clone https://github.com/toMyLord/DistributedFileParsingSystem.git 229 | cd DistributedFileParsingSystem 230 | cmake . 231 | make 232 | ``` 233 | 234 | ![编译过程](pictures/编译1.png) 235 | ![编译过程](pictures/编译2.png) 236 | 237 | 编译完成后会生成四个可执行文件：TaskDistribution、FileParsing、WorkStation、ManagerPlatform。 238 | 239 | 运行过程如下所示： 240 | 首先分别运行每个可执行程序，其中运行两个FileParsing程序（上线两个文件解析服务器）： 241 | ![初始化](pictures/初始化_2fp.png) 242 | 在WorkStation程序下输入需要发送的文件目录： 243 | ![第一次解析](pictures/第一次解析_2fp.png) 244 | 会发现第一个上线的文件解析服务器已经完成了解析任务，并且管理平台有相应的状态变化（时间太短，可以通过管理平台的log信息观察得到），同时任务分配服务器收到相应的解析完成信号。我们在WorkStation程序下再次输入需要发送的文件目录，再次发送文件： 245 | ![第二次解析](pictures/第二次解析_2fp.png) 246 | 我们可以观察到后上线的文件解析服务器在处理解析任务，并已经解析完成。同时任务分配服务器、管理平台也有相应的变化。我们继续上线一个文件解析服务器： 247 | ![新上线文件解析服务器](新上线fp_3fp.png) 248 | 可以在任务分配服务器的log中看到有新上线的文件解析服务器，同时在管理平台也可以看到对应的文件解析服务器。我们通过工作站连续发送三次文件。 249 | ![三文解析服务器第一次解析](pictures/3fp第一次解析.png) 250 | ![三文解析服务器第二次解析](pictures/3fp第二次解析.png) 251 | ![三文解析服务器第三次解析](pictures/3fp第三次解析.png) 252 | 我们可以观察到，处理解析任务的顺序依次是最先上线的文件解析服务器、第二个上线的文件解析服务器以及最新上线的文件解析服务器，符合预期。同时管理平台的界面显示和log显示、任务分配服务器的log显示也符合预期。我们再打开file_parse.log查看日志记录情况： 253 | ![日志](pictures/文件解析结果log.png) 254 | 日志内容也符合预期，整个系统成功运行。 255 | 256 | ## ChangeLog 257 | 258 | ### V1.0 2019-11-06 259 | 260 | 实现了整个分布式任务解析服务器框架，整个项目可以正常实现需求功能。 261 | 缺陷：任务分配服务器存在不能正常处理所有连接的正常close。 262 | 263 | ### V2.0 2019-11-11 264 | 265 | 在任务分配服务器处新增了心跳包机制——任务分配服务器会每隔5s向所有连接在线的文件解析服务器和工作站发送心跳包，文件解析服务器和工作站在收到心跳包后，也会主动回应心跳。通过心跳包机制可以很好的处理来自其他连接的断开情况。同时新增了在管理平台退出后，任务分配服务器可以重新接受来自管理平台的连接。 266 | -------------------------------------------------------------------------------- /taskdistribution/TaskDistribution_main.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mylord on 2019/10/31. 3 | // 4 | #include "src/distribution_server.h" 5 | #include 6 | #include 7 | #include 8 | 9 | typedef struct { 10 | int parsing_fd; 11 | }ReadyParsingNode; 12 | 13 | DistributionServer workstation_server(5555, 10); 14 | DistributionServer parsing_server(6666, 10); 15 | Server manager_server(7777, 1); 16 | bool is_manager_connected = false; //记录管理平台是否连接 17 | int manager_fd = 0; //管理平台客户端的fd 18 | queue ready_parsing_queue; //需要加互斥锁 19 | mutex queue_mutex; //给就绪队列加的互斥锁 20 | struct epoll_event fp_ev, fp_events[10]; //声明epoll_event结构体的变量,ev用于注册事件,数组用于回传要处理的事件 21 | struct epoll_event ws_ev, ws_events[10]; //声明epoll_event结构体的变量,ev用于注册事件,数组用于回传要处理的事件 22 | int fp_epoll_fd, ws_epoll_fd; 23 | SpecificTime st; 24 | 25 | 26 | void SendFpInfo(); 27 | 28 | void SendWsInfo(); 29 | 30 | void FpHandler_t(); 31 | 32 | void WsHandler_t(); 33 | 34 | void ParseRequestHandler_t(const int workstation_fd); 35 | 36 | int main() { 37 | struct epoll_event ev, events[4]; //声明epoll_event结构体的变量,ev用于注册事件,数组用于回传要处理的事件 38 | int listen_epoll_fd = epoll_create(4); //创建一个epoll的句柄，并告诉内核这个监听的数目为3 39 | int nfds; //记录需要处理的事件数 40 | ClientInfo manager_info; //管理平台客户端的信息 41 | bool isFpThreaded = false; //标记接受心跳线程是否建立 42 | bool isWsThreaded = false; //标记接受心跳线程是否建立 43 | 44 | fp_epoll_fd = epoll_create(10); 45 | ws_epoll_fd = epoll_create(10); 46 | 47 | ev.events = EPOLLIN | EPOLLET; //linsten_fd可读，边缘触发 48 | fp_ev.events = EPOLLIN; //linsten_fd可读，水平触发 49 | ws_ev.events = EPOLLIN; //linsten_fd可读，水平触发 50 | 51 | //将用于接受工作站信息的连接监听fd加入epoll池中 52 | ev.data.fd = workstation_server.getListenFd(); 53 | epoll_ctl(listen_epoll_fd, EPOLL_CTL_ADD, workstation_server.getListenFd(), &ev); 54 | cout << "[" << st.getTime().c_str() << " Listening WorkStation]:\tListen to WorkStation at port 5555." << endl; 55 | 56 | //将用于接受文件解析服务器信息的连接监听fd加入epoll池中 57 | ev.data.fd = parsing_server.getListenFd(); 58 | epoll_ctl(listen_epoll_fd, EPOLL_CTL_ADD, parsing_server.getListenFd(), &ev); 59 | cout << "[" << st.getTime().c_str() << " Listening Parser]:\tListen to Parser at port 6666." << endl; 60 | 61 | //将用于接受管理平台的连接监听fd加入epoll池中 62 | ev.data.fd = manager_server.getListenFd(); 63 | epoll_ctl(listen_epoll_fd, EPOLL_CTL_ADD, manager_server.getListenFd(), &ev); 64 | cout << "[" << st.getTime().c_str() << " Listening Manager]:\tListen to Manager at port 7777." << endl; 65 | 66 | while(true) { 67 | nfds = epoll_wait(listen_epoll_fd, events, 10, -1); 68 | for(int i = 0; i < nfds; i++){ 69 | if(events[i].data.fd == workstation_server.getListenFd()) { 70 | //有工作站连接至任务分配服务器 71 | //开线程监视一个完整的文件解析过程，接收来自工作站的解析请求以及接收来自文件解析服务器的解析完成回应都需要用阻塞的read 72 | ClientNode workstation_node; 73 | int workstation_fd = workstation_server.AcceptConnection(workstation_node); 74 | 75 | //将新连接的工作站加入epoll池 76 | ws_ev.data.fd = workstation_fd; 77 | epoll_ctl(ws_epoll_fd, EPOLL_CTL_ADD, workstation_fd, &ws_ev); 78 | 79 | if (isWsThreaded == false) { 80 | isWsThreaded = true; 81 | thread ws_t(WsHandler_t); 82 | ws_t.detach(); 83 | } 84 | 85 | cout << "[" << st.getTime().c_str() << " Connected WorkStation]:\tWorkstation " 86 | << workstation_node.client_info.client_ip << " is connected." << endl; 87 | 88 | 89 | if(is_manager_connected == true) { 90 | //通知管理平台有新上线的工作站.信息在workstation_node中 91 | SendWsInfo(); 92 | } 93 | } else if(events[i].data.fd == parsing_server.getListenFd()) { 94 | //有文件解析服务器连接至任务分配服务器 95 | ClientNode parsing_node; 96 | int parsing_fd = parsing_server.AcceptConnection(parsing_node); 97 | 98 | //当有文件解析服务器加入任务分配服务器时，直接将其加入就绪队列 99 | ReadyParsingNode temp; 100 | temp.parsing_fd = parsing_fd; 101 | 102 | queue_mutex.lock(); 103 | ready_parsing_queue.push(temp); 104 | queue_mutex.unlock(); 105 | 106 | //将新连接的文件解析服务器加入epoll池 107 | fp_ev.data.fd = parsing_fd; 108 | epoll_ctl(fp_epoll_fd, EPOLL_CTL_ADD, parsing_fd, &fp_ev); 109 | 110 | if (isFpThreaded == false) { 111 | isFpThreaded = true; 112 | thread fp_t(FpHandler_t); 113 | fp_t.detach(); 114 | } 115 | 116 | cout << "[" << st.getTime().c_str() << " Connected Parser]:\tParser " 117 | << parsing_node.client_info.client_ip << " is connected and added to queue." << endl; 118 | 119 | if(is_manager_connected == true) { 120 | //通知管理平台有新上线的文件解析服务器,信息在parsing_node中 121 | SendFpInfo(); 122 | } 123 | } else if(events[i].data.fd == manager_server.getListenFd()) { 124 | //有文件管理平台连接至任务分配服务器 125 | //谨记：管理平台只能连接一个!!! 126 | if(is_manager_connected == false) { 127 | //如果管理平台还未连接，则连接管理平台 128 | manager_fd = manager_server.AcceptConnection(manager_info); 129 | 130 | is_manager_connected = true; 131 | 132 | //将manager platform加入listenepoll池，用来检测是否主动断开 133 | ev.data.fd = manager_fd; 134 | epoll_ctl(listen_epoll_fd, EPOLL_CTL_ADD, manager_fd, &ev); 135 | 136 | cout << "[" << st.getTime().c_str() << " Connected Manager]:\tManager " 137 | << manager_info.client_ip << " is connected to Task Distributer." << endl; 138 | 139 | //连接后需要向管理平台发送已经上线的所有工作站及文件解析服务器的信息 140 | SendFpInfo(); 141 | usleep(1); 142 | SendWsInfo(); 143 | } 144 | } else if(events[i].data.fd == manager_fd && manager_fd != 0) { 145 | char buffer[MAX_BUFFER_SIZE]; 146 | if(manager_server.Read(manager_fd, buffer) == 0) { 147 | //管理平台断开 148 | manager_server.Close(manager_fd); 149 | 150 | is_manager_connected = false; 151 | 152 | cout << "[" << st.getTime().c_str() << 153 | " Manager Disconnected]:\tManager platform disconnected from distributer server." << endl; 154 | } else { 155 | cout << "[" << st.getTime().c_str() << 156 | " Manager Info]:\tReceived information from Manager platform :" << buffer << endl; 157 | } 158 | } else { 159 | cout << "[" << st.getTime().c_str() << " Epoll Error]:\tNo corresponding Epoll event was found." << endl; 160 | } 161 | } 162 | } 163 | } 164 | 165 | 166 | void SendFpInfo() { 167 | char sender[1024]; 168 | string buff("fp:"); 169 | parsing_server.getAllClientInfo(buff); 170 | strcpy(sender, buff.c_str()); 171 | manager_server.Write(manager_fd, sender); 172 | } 173 | 174 | 175 | void SendWsInfo() { 176 | char sender[1024]; 177 | string buff("ws:"); 178 | workstation_server.getAllClientInfo(buff); 179 | strcpy(sender, buff.c_str()); 180 | manager_server.Write(manager_fd, sender); 181 | } 182 | 183 | 184 | void FpHandler_t() { 185 | char buffer[MAX_BUFFER_SIZE]; 186 | 187 | while(true) { 188 | int nfds = epoll_wait(fp_epoll_fd, fp_events, 10, -1); 189 | for (int i = 0; i < nfds; i++) { 190 | if (parsing_server.Read(fp_events[i].data.fd, buffer) == 0) { 191 | //收到close请求 192 | parsing_server.Close(fp_events[i].data.fd); 193 | SendFpInfo(); 194 | 195 | //从epoll池中删除该fd 196 | fp_ev.data.fd = fp_events[i].data.fd; 197 | epoll_ctl(fp_epoll_fd, EPOLL_CTL_DEL, fp_events[i].data.fd, &fp_ev); 198 | 199 | cout << "[" << st.getTime().c_str() <<" Parser Close]:\tParser client exit! ip:" 200 | << parsing_server.getClientNode( fp_events[i].data.fd).client_info.client_ip 201 | << "\tfd:" << parsing_server.getClientNode( fp_events[i].data.fd).client_info.client_fd 202 | << endl; 203 | 204 | continue; 205 | } 206 | if (strncmp(buffer, "HeartBeats", 10) == 0) { 207 | //收到心跳包 208 | // cout << "heartbeats by parser!" << endl; 209 | 210 | } else if (strncmp(buffer, "ParsingSuccess", 14) == 0) { 211 | //收到成功解析回执 212 | ReadyParsingNode temp; 213 | 214 | temp.parsing_fd = fp_events[i].data.fd; 215 | 216 | queue_mutex.lock(); 217 | ready_parsing_queue.push(temp); 218 | queue_mutex.unlock(); 219 | 220 | parsing_server.setState(fp_events[i].data.fd, WAITING); 221 | 222 | cout << "[" << st.getTime().c_str() << 223 | " Parsing Success]:\tParse file succeed!" << endl; 224 | 225 | //向管理平台发送解析完成信息 226 | SendFpInfo(); 227 | } else { 228 | //无法解析接收内容 229 | cout << "[" << st.getTime().c_str() << 230 | " Reply Error]:\tTask distribution server cann't parse the reply information from parser : " << 231 | buffer << endl; 232 | } 233 | } 234 | } 235 | } 236 | 237 | 238 | void WsHandler_t() { 239 | char buffer[MAX_BUFFER_SIZE]; 240 | 241 | while(true) { 242 | int nfds = epoll_wait(ws_epoll_fd, ws_events, 10, -1); 243 | for (int i = 0; i < nfds; i++) { 244 | if (workstation_server.Read(ws_events[i].data.fd, buffer) == 0) { 245 | //收到close请求 246 | workstation_server.Close(ws_events[i].data.fd); 247 | SendWsInfo(); 248 | 249 | //从epoll池中删除该fd 250 | ws_ev.data.fd = ws_events[i].data.fd; 251 | epoll_ctl(ws_epoll_fd, EPOLL_CTL_DEL, ws_events[i].data.fd, &ws_ev); 252 | 253 | cout << "[" << st.getTime().c_str() <<" Workstation Close]:\tWorkStation client exit! ip:" 254 | << workstation_server.getClientNode( ws_events[i].data.fd).client_info.client_ip 255 | << "\tfd:" << workstation_server.getClientNode( ws_events[i].data.fd).client_info.client_fd 256 | << endl; 257 | 258 | continue; 259 | } 260 | if (strncmp(buffer, "HeartBeats", 10) == 0) { 261 | //收到心跳包 262 | // cout << "heartbeats by workstation!" << endl; 263 | 264 | } else if (strncmp(buffer, "TransmitSuccess", 15) == 0) { 265 | //收到成功解析回执 266 | 267 | workstation_server.setState(ws_events[i].data.fd, WAITING); 268 | 269 | cout << "[" << st.getTime().c_str() << 270 | " Transmit Success]:\tWorkStation transmit files succeed!" << endl; 271 | 272 | SendWsInfo(); 273 | } else if (strncmp(buffer, "ParsingRequest", 14) == 0) { 274 | //有来自工作站的解析请求。可能存在就绪队列空而导致长时间等待的情况因此要用线程的方式处理解析请求。 275 | int workstation_fd = ws_events[i].data.fd; 276 | thread parse_t(ParseRequestHandler_t, workstation_fd); 277 | parse_t.detach(); 278 | } else { 279 | //无法解析接收内容 280 | cout << "[" << st.getTime().c_str() << 281 | " Reply Error]:\tTask distribution server cann't parse the reply information from workstation : " << 282 | buffer << endl; 283 | } 284 | } 285 | } 286 | } 287 | 288 | 289 | void ParseRequestHandler_t(const int workstation_fd) { 290 | char buffer[MAX_BUFFER_SIZE]; 291 | 292 | //对发送来的数据进行解析并根据就绪队列向文件解析服务器发送连接命令，如果就绪队列为空，就等待1s 293 | cout << "[" << st.getTime().c_str() << 294 | " Request Received]:\tParsing request was received, start distributing!" << endl; 295 | 296 | while(true) { 297 | queue_mutex.lock(); 298 | if(ready_parsing_queue.empty() != 1) //如果就绪队列不空，先不拿掉锁，等pop之后再取锁 299 | break; 300 | queue_mutex.unlock(); 301 | sleep(1); 302 | } 303 | 304 | int parsing_fd = ready_parsing_queue.front().parsing_fd; 305 | ready_parsing_queue.pop(); 306 | queue_mutex.unlock(); 307 | 308 | ClientNode workstation_node = workstation_server.getClientNode(workstation_fd); 309 | //构造好接口，buffer内需要有工作站的ip信息,端口默认=8888 310 | int lenth = workstation_node.client_info.client_ip.length(); 311 | workstation_node.client_info.client_ip.copy(buffer, lenth); 312 | buffer[lenth] = '\0'; 313 | parsing_server.Write(parsing_fd, buffer); 314 | 315 | workstation_server.setState(workstation_fd, TRANSMITING); 316 | parsing_server.setState(parsing_fd, TRANSMITING); 317 | 318 | //向管理平台发送正在解析文件信息 319 | SendFpInfo(); 320 | usleep(1); 321 | SendWsInfo(); 322 | } --------------------------------------------------------------------------------