├── .gitignore
├── ACC-Runtime-Hook
    ├── .gitignore
    ├── README.md
    ├── Make.Dockerfile
    ├── Makefile
    ├── main.go
    └── container_config.go
├── ACC-Runtime
    ├── README.md
    ├── .gitignore
    ├── Make.Dockerfile
    ├── Makefile
    └── acc-runtime.patch
├── ACC-Manager
    ├── .gitignore
    ├── test
    │   └── tb_device-detector.cc
    ├── include
    │   ├── driver-manager.hpp
    │   ├── device-detector.hpp
    │   ├── manager.hpp
    │   ├── mounter.hpp
    │   ├── cli.hpp
    │   ├── device-parser.hpp
    │   ├── utils.hpp
    │   └── common.hpp
    ├── src
    │   ├── driver-manager.cc
    │   ├── device-detector.cc
    │   ├── release.cc
    │   ├── cli.cc
    │   ├── list.cc
    │   ├── acc-manager.cc
    │   ├── configure.cc
    │   ├── manager.cc
    │   ├── common.cc
    │   ├── utils.cc
    │   ├── mounter.cc
    │   └── device-parser.cc
    ├── Make.Dockerfile
    ├── README.md
    ├── out
    │   ├── device.proto
    │   └── device.pbtxt
    └── Makefile
├── images
    └── Accelerator-Docker.png
├── daemon.json
├── LICENSE
├── Makefile
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
1 | out/
2 | 
3 | 


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/.gitignore:
--------------------------------------------------------------------------------
1 | out/
2 | 
3 | 


--------------------------------------------------------------------------------
/ACC-Runtime/README.md:
--------------------------------------------------------------------------------
1 | # ACC-Runtime
2 | 


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/README.md:
--------------------------------------------------------------------------------
1 | # ACC-Runtime-Hook
2 | 


--------------------------------------------------------------------------------
/ACC-Runtime/.gitignore:
--------------------------------------------------------------------------------
1 | out/
2 | .tmp_go_path/
3 | 
4 | 


--------------------------------------------------------------------------------
/ACC-Manager/.gitignore:
--------------------------------------------------------------------------------
1 | out/
2 | include/device.pb.h
3 | src/device.pb.cc
4 | 


--------------------------------------------------------------------------------
/images/Accelerator-Docker.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/icns-distributed-cloud/ACCEL-docker/HEAD/images/Accelerator-Docker.png


--------------------------------------------------------------------------------
/daemon.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "default-runtime": "acc-runtime",
 3 |     "runtimes": {
 4 |         "acc-runtime": {
 5 |             "path": "/usr/bin/acc-runtime",
 6 |             "runtimeArgs": []
 7 |         }
 8 |     }
 9 | }
10 | 


--------------------------------------------------------------------------------
/ACC-Manager/test/tb_device-detector.cc:
--------------------------------------------------------------------------------
 1 | #include "device-detector.hpp"
 2 | #include "common.hpp"
 3 | #include <iostream>
 4 | 
 5 | using namespace std;
 6 | 
 7 | int main(int argc, char *argv[]){
 8 | 
 9 |     /*DeviceParser p_dev(argv[1]);
10 |     list<Device> devs;
11 |     devs = p_dev.parse();
12 | 
13 |     list<Device>::iterator iter;
14 |     for(iter = devs.begin(); iter!= devs.end(); iter++){
15 |                 cout << iter-> getName() << endl;
16 |                 cout << iter-> getPciSlot() << endl;
17 |     }*/
18 | 
19 | }
20 | 


--------------------------------------------------------------------------------
/ACC-Manager/include/driver-manager.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_DRIVER_MANAGER
 2 | #define _ACC_MANAGER_DRIVER_MANAGER
 3 | 
 4 | #include "common.hpp"
 5 | 
 6 | // For switching device drivers --> not used yet
 7 | class DriverManager{
 8 |     public:
 9 |         DriverManager();
10 |         bool switchDriver(Device device, Driver target);
11 |         list<Device> getAllDevices();
12 |         list<Driver> getAllDrivers();
13 |     private:
14 |         list<Device> devices;
15 |         list<Driver> drivers;
16 | };
17 | 
18 | #endif


--------------------------------------------------------------------------------
/ACC-Manager/src/driver-manager.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | #include "driver-manager.hpp"
 3 | 
 4 | // Driver Manager
 5 | // : Handles device driver switch --> not implemented yet
 6 | DriverManager::DriverManager(){}
 7 | bool DriverManager::switchDriver(Device device, Driver target){
 8 |     return true;
 9 | }
10 | list<Device> DriverManager::getAllDevices(){
11 |     return devices;
12 | }
13 | list<Driver> DriverManager::getAllDrivers(){
14 |     return drivers;
15 | }
16 | 
17 | bool driver_parse(int argc, char** argv, Context *ctx){
18 | 
19 | }
20 | 
21 | bool driver_command(Context *ctx){
22 | 
23 | }


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/Make.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM ubuntu:16.04
 2 | 
 3 | ARG UNAME=tmpuser
 4 | ARG UID=1000
 5 | ARG GID=1000
 6 | RUN groupadd -g $GID -o $UNAME && \
 7 |     useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
 8 | 
 9 | RUN mkdir /ACC-Runtime-Hook && \
10 |     apt-get update && \
11 |     apt-get install -y build-essential wget && \
12 |     wget -qO- https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz | tar xvz -C /usr/local
13 | 
14 | ENV PATH $PATH:/usr/local/go/bin
15 | 
16 | COPY . /ACC-Runtime-Hook
17 | RUN chown -R $UNAME:$UNAME /ACC-Runtime-Hook
18 | 
19 | USER $UNAME
20 | 
21 | RUN cd /ACC-Runtime-Hook && \
22 |     make
23 | 


--------------------------------------------------------------------------------
/ACC-Runtime/Make.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM ubuntu:16.04
 2 | 
 3 | ARG UNAME=tmpuser
 4 | ARG UID=1000
 5 | ARG GID=1000
 6 | RUN groupadd -g $GID -o $UNAME && \
 7 |     useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
 8 | 
 9 | RUN mkdir /ACC-Runtime && \
10 |     apt-get update && \
11 |     apt-get install -y build-essential wget git pkg-config libseccomp-dev libapparmor-dev libselinux1-dev && \
12 |     wget -qO- https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz | tar xvz -C /usr/local
13 | 
14 | ENV PATH $PATH:/usr/local/go/bin
15 | 
16 | COPY . /ACC-Runtime
17 | RUN chown -R $UNAME:$UNAME /ACC-Runtime
18 | 
19 | USER $UNAME
20 | 
21 | RUN cd /ACC-Runtime && \
22 |     make
23 | 


--------------------------------------------------------------------------------
/ACC-Manager/include/device-detector.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_DEVICE_DETECTOR
 2 | #define _ACC_MANAGER_DEVICE_DETECTOR
 3 | 
 4 | #include <iostream>
 5 | #include <fcntl.h>
 6 | #include <fstream>
 7 | #include <string>
 8 | #include <pci/pci.h>
 9 | #include <google/protobuf/text_format.h>
10 | #include <google/protobuf/io/zero_copy_stream_impl.h>
11 | #include "device.pb.h"
12 | #include "common.hpp"
13 | #include "utils.hpp"
14 | 
15 | // Device auto-detector --> not implemented yet
16 | class DeviceDetector{
17 |     public:
18 |         DeviceDetector();
19 |         bool detect(list<Accelerator>*);
20 |         bool detect(list<Accelerator>*, list<Accelerator>);
21 |     private:
22 | };
23 | #endif
24 | 


--------------------------------------------------------------------------------
/ACC-Manager/Make.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM ubuntu:16.04
 2 | 
 3 | ARG UNAME=tmpuser
 4 | ARG UID=1000
 5 | ARG GID=1000
 6 | RUN groupadd -g $GID -o $UNAME && \
 7 |     useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
 8 | 
 9 | RUN mkdir /ACC-Manager && \
10 |     apt-get update && \
11 |     apt-get install -y wget autoconf automake libtool curl make g++ unzip libcap-dev libpci-dev && \
12 |     wget -qO- https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz | tar xvz -C / && \
13 |     cd /protobuf-3.6.1 && \
14 |     ./configure && \
15 |     make && \
16 |     make check && \
17 |     make install && \
18 |     ldconfig
19 | 
20 | COPY . /ACC-Manager
21 | RUN chown -R $UNAME:$UNAME /ACC-Manager
22 | 
23 | USER $UNAME
24 | 
25 | RUN cd /ACC-Manager && \
26 |     make
27 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/device-detector.cc:
--------------------------------------------------------------------------------
 1 | #include "device-detector.hpp"
 2 | 
 3 | using namespace std;
 4 | using namespace google;
 5 | 
 6 | // Device Detector
 7 | // : Auto-detect devices using PCI-e bus info., etc.
 8 | // not implemented yet
 9 | DeviceDetector::DeviceDetector() = default;
10 | 
11 | // Detect and move detected device list to 'to'
12 | bool DeviceDetector::detect(list<Accelerator>* to){
13 |     list<Accelerator> accs;
14 |     //TODO : implement detect method
15 |     *to = accs;
16 |     return true;
17 | }
18 | 
19 | // Detect and append detected device list to '_acc' and move it to 'to'
20 | bool DeviceDetector::detect(list<Accelerator>* to, list<Accelerator> _acc){
21 |     list<Accelerator> accs;
22 |     detect(&accs);
23 |     _acc.insert(_acc.end(),accs.begin(),accs.end());
24 |     *to = _acc;
25 |     return true;
26 | }


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/Makefile:
--------------------------------------------------------------------------------
 1 | .NOTPARALLEL:
 2 | .PHONY:all
 3 | 
 4 | SOURCES := $(shell find . 2>&1 | grep -E '.*\.(c|h|go)$$')
 5 | 
 6 | OUT_DIR := ./out
 7 | BIN_NAME := acc-runtime-hook
 8 | 
 9 | all: $(SOURCES)
10 | 	@if [ ! -d "$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi	
11 | 	go build -ldflags "-s -w -v" -o $(OUT_DIR)/$(BIN_NAME) .
12 | 
13 | cont-ubuntu16.04:
14 | 	@if [ ! -d "$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi
15 | 	docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) -t "acc/hook/ubuntu:16.04" -f Make.Dockerfile .
16 | 	docker run --rm -v $(CURDIR)/$(OUT_DIR):/dist:Z -u $(shell id -u):$(shell id -g) "acc/hook/ubuntu:16.04" /bin/bash -c "cp /ACC-Runtime-Hook/out/acc-runtime-hook /dist"
17 | 
18 | install: 
19 | 	cp $(OUT_DIR)/${BIN_NAME} /usr/bin/${BIN_NAME}
20 | 
21 | clean:
22 | 	rm -rf $(OUT_DIR)
23 | 	if [ "$(shell docker images -q acc/hook/ubuntu:16.04 2> /dev/null)" != "" ]; then docker rmi "acc/hook/ubuntu:16.04"; fi
24 | 
25 | 


--------------------------------------------------------------------------------
/ACC-Manager/README.md:
--------------------------------------------------------------------------------
 1 | # ACC-Manager
 2 | 
 3 | ACC-Manager provides information of all registered accelerators.
 4 | acc manager list shows all accelerators and their status.
 5 | If an accelerator is used by a container, it print the container information using the hardware as "holder".
 6 | 
 7 | ```
 8 | $ acc-manager list
 9 | ```
10 | 
11 | 
12 | ```
13 | +------------+--------------+------------------------+----------------+----------------+----------+
14 | |     ID     |     Name     |          Type          |    PCI-Slot    |     Status     |  Holder  |
15 | +------------+--------------+------------------------+----------------+----------------+----------+
16 | | 234750     | QuadroM2000  | nvidia.com/gpu         | 0000:02:00.0   | Available      | 0        |
17 | | 1121730    | KCU-1500     | xilinx.fpga/kcu-1500   | 0000:01:00.0   | Available      | 0        |
18 | +------------+--------------+------------------------+----------------+----------------+----------+
19 | ```
20 | 
21 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/release.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | #include "manager.hpp"
 3 | 
 4 | // Release format
 5 | // : acc-manager release --pid=<PID>
 6 | bool release_parse(int argc, char** argv, Context* ctx)
 7 | {
 8 |     for (int i = 0; i < argc; i++) {
 9 |         cmatch cm;
10 |         regex r_pid("--pid=([0-9]*)");
11 |         // Parse PID
12 |         if (regex_match(argv[i], cm, r_pid) && cm.size() > 1) {
13 |             ctx->setPid( atoi(cm[1].str().c_str()) );
14 |         }
15 |     }
16 |     if(ctx->getPid() < 0){
17 |         errx(ERR_CODE_PID_NOT_VALID,"Not valid pid");
18 |         return false;
19 |     }
20 |     return true;
21 | }
22 | 
23 | // Release all devices by calling manager
24 | bool release_command(Context* ctx)
25 | {
26 |     Manager manager(ctx);
27 |     if (!manager.releaseDevices()){
28 |         errx(ERR_CODE_RELEASE_DEVICE_FAILED,"Releasing devices failed");
29 |         return false;
30 |     }
31 |     return true;
32 | }


--------------------------------------------------------------------------------
/ACC-Manager/out/device.proto:
--------------------------------------------------------------------------------
 1 | syntax = "proto2";
 2 | 
 3 | package device;
 4 | 
 5 | message env_entry {
 6 |     required string key = 1;
 7 |     required string val = 2;
 8 | }
 9 | message file_entry {
10 |     required string src = 1;
11 |     required string dst = 2;
12 | }
13 | message device {
14 |     required string name = 1;
15 |     optional string pci = 2;
16 |     repeated string device_file = 3;
17 |     repeated string library = 4;
18 |     repeated file_entry file = 5;
19 |     repeated env_entry env = 6;
20 |     optional Status status = 7;
21 |     optional int32 pid = 8;
22 |     optional string id = 9;
23 |     enum Status {
24 |         IDLE = 0;
25 |         USED = 1;
26 |         MISS = 2;
27 |     }
28 | }
29 | 
30 | message accelerator {
31 |     required string type = 1;
32 |     repeated device devices = 2;
33 | }
34 | 
35 | message device_list {
36 |     repeated device devices = 1;
37 | }
38 | 
39 | message accelerator_list {
40 |     repeated accelerator accelerators = 1;
41 | }


--------------------------------------------------------------------------------
/ACC-Manager/include/manager.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_MANAGER
 2 | #define _ACC_MANAGER_MANAGER
 3 | 
 4 | #include "utils.hpp"
 5 | #include "mounter.hpp"
 6 | #include "device-detector.hpp"
 7 | #include "device-parser.hpp"
 8 | 
 9 | #define ERR_CODE_DEVICE_LOCKED 10
10 | #define ERR_CODE_DEVICE_NOT_FOUND 11
11 | #define ERR_CODE_DEVICE_MISCONFIGURED 14
12 | #define ERR_CODE_PARSE_DETECT_DEVICES_FAILED 12
13 | #define ERR_CODE_RELEASE_DEVICE_FAILED 13
14 | 
15 | // Manager handling request/release of device on container
16 | class Manager{
17 |     public:
18 |         explicit Manager(Context*);
19 |         bool requestDevices();
20 |         bool requestDevice(Device&);
21 |         bool releaseDevices();
22 | 
23 |         list<Accelerator> getAcceleratorList();
24 |     private:
25 |         Context *ctx;
26 |         string status_file;
27 |         list<Accelerator> acc_list;
28 | 
29 |         bool updateStatusFile();
30 |         bool parseAndDetectAccelerators();
31 | };
32 | 
33 | #endif


--------------------------------------------------------------------------------
/ACC-Manager/include/mounter.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_MOUNTER
 2 | #define _ACC_MANAGER_MOUNTER
 3 | 
 4 | #include "utils.hpp"
 5 | 
 6 | #include <sys/mount.h>
 7 | #include <sys/types.h>
 8 | 
 9 | #include <sched.h>
10 | #include <unistd.h>
11 | #include <fcntl.h>
12 | #include <libgen.h>
13 | #include <fstream>
14 | 
15 | #define ERR_CODE_ONLY_ABS_PATH_ALLOWED 31
16 | 
17 | // Mounter class for mounting needed files to container
18 | class Mounter{
19 |     public:
20 |         explicit Mounter(Context*);
21 |         bool mountDevices(list<Device>);
22 |         bool mountDevice(Device);
23 |     private:
24 |         Context* cont;
25 |         bool mountDeviceFiles(list<string>);
26 |         bool mountDeviceFile(string, string);
27 |         bool mountLibraries(list<string>);
28 |         bool mountLibrary(string);
29 |         bool mountFiles(list<array<string,2>>);
30 |         bool mountFile(string,string);
31 |         bool setEnvs(list<array<string,2>>);
32 | 
33 |         bool createDev(string,struct stat);
34 | };
35 | 
36 | #endif


--------------------------------------------------------------------------------
/ACC-Manager/include/cli.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_CLI
 2 | #define _ACC_MANAGER_CLI
 3 | 
 4 | #include <algorithm>
 5 | #include <regex>
 6 | #include "utils.hpp"
 7 | 
 8 | #define EXIT_CODE_SUCCESS 0
 9 | #define ERR_CODE_PARSE_ERROR 1
10 | #define ERR_CODE_FUNC_ERROR 2
11 | #define ERR_CODE_COMMAND_ERROR 3
12 | 
13 | class Context;
14 | 
15 | //extern const struct argp configure_usage;
16 | bool configure_parse(int, char**, Context *);
17 | bool configure_command(Context *);
18 | 
19 | //extern const struct argp release_usage;
20 | bool release_parse(int, char**, Context *);
21 | bool release_command(Context *);
22 | 
23 | //extern const struct argp list_usage;
24 | bool list_parse(int, char**, Context *);
25 | bool list_command(Context *);
26 | 
27 | //print help instructions
28 | bool help_parse(int, char**, Context *);
29 | bool help_command(Context *);
30 | 
31 | class Command {
32 |     public:
33 |         Command(string name, bool (*)(int, char**,Context*), bool (*)(Context*));
34 |         string name;
35 |         bool (*parse_fn)(int argc, char** argv,Context*);
36 |         bool (*func)(Context *ctx);
37 | };
38 | 
39 | #endif
40 | 


--------------------------------------------------------------------------------
/ACC-Manager/out/device.pbtxt:
--------------------------------------------------------------------------------
 1 | accelerators:[{
 2 |     type: "intel.fpga/arria10gx_devkit",
 3 |     devices: [{
 4 |         name: "arria10gx-devkit-0",
 5 |         pci: "0000:05:00.0",
 6 |         device_file:[
 7 |             "/dev/acla10_vfpga0"
 8 |         ],
 9 |         library:[
10 |             "/home/ncl/intelFPGA_pro/17.1/hld/host/linux64/lib/libOpenCL.so.1",
11 |             "/home/ncl/intelFPGA_pro/17.1/hld/host/linux64/lib/libalteracl.so",
12 |             "/home/ncl/intelFPGA_pro/17.1/hld/host/linux64/lib/libacl_emulator_kernel_rt.so",
13 |             "/home/ncl/intelFPGA_pro/17.1/hld/host/linux64/lib/libelf.so.0"
14 |         ],
15 |         file:[{
16 |             src: "/home/ncl/intelFPGA_pro/17.1/hld/board/a10_ref/linux64/lib/libaltera_a10_ref_mmd.so",
17 |             dst: "/home/ncl/intelFPGA_pro/17.1/hld/board/a10_ref/linux64/lib/libaltera_a10_ref_mmd.so"
18 |         },{
19 |             src: "/etc/OpenCL/vendors/Altera.icd",
20 |             dst: "/etc/OpenCL/vendors/Altera.icd"
21 |         },{
22 |             src: "/opt/Intel/OpenCL/Boards/a10_ref.fcd",
23 |             dst: "/opt/Intel/OpenCL/Boards/a10_ref.fcd"
24 |         }],
25 |         env:[{
26 |             key: "EXAMPLE",
27 |             val: "val"
28 |         }]
29 |     }]
30 | }]
31 | 


--------------------------------------------------------------------------------
/ACC-Runtime/Makefile:
--------------------------------------------------------------------------------
 1 | .PHONY:all
 2 | 
 3 | TMP_GO_PATH := .tmp_go_path
 4 | OUT_DIR := ./out
 5 | 
 6 | TMP_GO_ABS := $(CURDIR)/$(TMP_GO_PATH)
 7 | ENV_GOPATH := env GOPATH=$(TMP_GO_ABS)
 8 | 
 9 | RUNC_REPO:=github.com/opencontainers/runc
10 | 
11 | TARGET := acc-runtime
12 | 
13 | all: $(clean) $(TARGET)
14 | 
15 | $(TARGET): 
16 | 	@if [ ! -d "$(TMP_GO_ABS)" ]; then mkdir $(TMP_GO_ABS); fi
17 | 	@echo Setting temp. GOPATH as [$(TMP_GO_ABS)]
18 | 	$(ENV_GOPATH) go get -d $(RUNC_REPO)/...
19 | 	patch -d$(TMP_GO_ABS)/src/$(RUNC_REPO) -p1 < acc-runtime.patch
20 | 	$(ENV_GOPATH) make -C $(TMP_GO_ABS)/src/$(RUNC_REPO)/ BUILDTAGS='apparmor seccomp'
21 | 	@if [ ! -d "$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi
22 | 	@mv $(TMP_GO_ABS)/src/$(RUNC_REPO)/$@ $(OUT_DIR)/$@
23 | 
24 | cont-ubuntu16.04:
25 | 	@if [ ! -d "$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi
26 | 	docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) -t "acc/runtime/ubuntu:16.04" -f Make.Dockerfile .
27 | 	docker run --rm -v $(CURDIR)/$(OUT_DIR):/dist:Z -u $(shell id -u):$(shell id -g) "acc/runtime/ubuntu:16.04" /bin/bash -c "cp /ACC-Runtime/out/acc-runtime /dist"
28 | 
29 | clean: 
30 | 	rm -rf $(TMP_GO_PATH)
31 | 	rm -rf $(OUT_DIR)
32 | 	if [ "$(shell docker images -q acc/runtime/ubuntu:16.04 2> /dev/null)" != "" ]; then docker rmi "acc/runtime/ubuntu:16.04"; fi
33 | 
34 | install:
35 | 	cp $(OUT_DIR)/$(TARGET) /usr/bin/$(TARGET)
36 | 
37 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/cli.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | 
 3 | Command::Command(const string _name, bool (*_parse_fn)(int,char**,Context*), bool (*_func)(Context*)) : name(_name), parse_fn(_parse_fn), func(_func){}
 4 | 
 5 | bool help_parse(int argc, char** argv, Context *ctx){
 6 |     return true;
 7 | }
 8 | bool help_command(Context *ctx){
 9 | 
10 |     printf("\n");
11 |     printf("+-------------------------------------------------------+\n");
12 |     printf("|              Accelerator-Docker  Manager              |\n");
13 |     printf("+-------------------------------------------------------+\n");
14 |     printf(" Accelerator-Docker provides ACC-Manager which gives \n information about status of accelerators.\n");
15 |     printf(" To execute the manager, please enter the option name.\n");
16 |     printf("+-------------------------------------------------------+\n\n");
17 |     printf(" \t <how to use ACC-Manager > \n\n");
18 |     printf(" acc-manager [OPTION1] \n\n");
19 |     printf(" [OPTION1] \n\n");
20 |     printf("    help:   print help instructions\n");
21 |     printf("    list:   print list of accelerators and their status \n\n");
22 |     printf("+-------------------------------------------------------+\n\n");
23 |     printf("\t <how to use Accelerator-Docker > \n\n");
24 |     printf(" docker run --runtime acc-runtime -e ACC_VISIBLE_DEVICES={NAME | ID}\n\n\n");
25 |     return true;
26 | }
27 | 


--------------------------------------------------------------------------------
/ACC-Manager/include/device-parser.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_DEVICE_PARSER
 2 | #define _ACC_MANAGER_DEVICE_PARSER
 3 | 
 4 | #ifdef VERBOSE
 5 | #define ACC_VERBOSE true
 6 | #else
 7 | #define ACC_VERBOSE false
 8 | #endif
 9 | 
10 | #include <iostream>
11 | #include <fcntl.h>
12 | #include <fstream>
13 | #include <string>
14 | #include <regex>
15 | #include <google/protobuf/text_format.h>
16 | #include <google/protobuf/io/zero_copy_stream_impl.h>
17 | #include "device.pb.h"
18 | #include "common.hpp"
19 | #include "utils.hpp"
20 | #include <sys/utsname.h>
21 | 
22 | // Device parser for parsing user-defined pass through configuration
23 | class DeviceParser{
24 | public:
25 |     DeviceParser(char*, char*);
26 |     bool parse(list<Accelerator>*);
27 |     bool parse(list<Accelerator>*,const char*);
28 | private:
29 |     char* statusFilePath;
30 |     char* deviceFilePath;
31 | 
32 |     list<Accelerator> protoToAccelerator(device::accelerator_list*);    // For user-specified file
33 |     list<Device> protoToDeviceList(const device::device_list*);            // For status file
34 |     Device protoToDevice(const device::device*);
35 |     map<string,Device> devListToDevMap(list<Device>);
36 |     bool isListValid(list<Accelerator>&);
37 |     bool isAcceleratorValid(Accelerator&);
38 |     bool isDeviceValid(Device&);
39 | 
40 |     set<string> deviceNamesTmp;
41 |     string ld_cache;
42 |     string machine_arch;
43 | };
44 | #endif
45 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/list.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | #include "manager.hpp"
 3 | 
 4 | // Nothing to do with parsing in list command
 5 | bool list_parse(int argc, char** argv, Context *ctx){
 6 |     return true;
 7 | }
 8 | 
 9 | // List command current status of each accelerator devices
10 | bool list_command(Context *ctx){
11 |     Manager manager(ctx);
12 |     list<Accelerator> accs = manager.getAcceleratorList();
13 | 
14 |     printf("+------------+------------------------+------------------------------+----------------+----------------+----------+\n");
15 |     printf("|     ID     |          Name          |             Type             |    PCI-Slot    |     Status     |  Holder  |\n");
16 |     printf("+------------+------------------------+------------------------------+----------------+----------------+----------+\n");
17 |     for(auto it_acc : accs){
18 |         list<Device> devices = it_acc.getDevices();
19 |         for(auto it : devices){
20 |             printf("| %-11s| %-23s| %-29s| %-15s| %-15s| %-9d|\n",
21 |                 it.getId().c_str(),
22 |                 it.getName().c_str(),
23 |                 it_acc.getType().c_str(),
24 |                 it.getPciSlot().c_str(),
25 |                 it.getStatus() == Device::Status::AVAILABLE ? "Available" : it.getStatus() == Device::Status::MISCONFIGURED ? "Misconfigured" : "Unavailable",
26 |                 it.getPid()
27 |             );
28 |         }
29 |     }
30 |     printf("+------------+------------------------+------------------------------+----------------+----------------+----------+\n");
31 |     return true;
32 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2018-2019, KAIST-NCL
 2 | All rights reserved.
 3 | 
 4 | Redistribution and use in source and binary forms, with or without modification,
 5 | are permitted provided that the following conditions are met:
 6 | 
 7 |  - Redistributions of source code must retain the above copyright notice,
 8 |    this list of conditions and the following disclaimer.
 9 | 
10 |  - Redistributions in binary form must reproduce the above copyright notice,
11 |    this list of conditions and the following disclaimer in the documentation
12 |    and/or other materials provided with the distribution.
13 | 
14 |  - Neither the name of Thomas J Bradley nor the names of its contributors may
15 |    be used to endorse or promote products derived from this software without
16 |    specific prior written permission.
17 | 
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/acc-manager.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | 
 3 | #define COMMAND_CNT 4
 4 | 
 5 | Command command_list[COMMAND_CNT] = {
 6 |     { "configure", &configure_parse, &configure_command },
 7 |     { "release", &release_parse, &release_command},
 8 |     { "list", &list_parse, &list_command },
 9 |     { "help", &help_parse, &help_command}
10 | };
11 | 
12 | //TODO : Error handling --> using exception
13 | //TODO : Set status file, device file path with given option '--config= , --stat='
14 | int main(int argc, char** argv)
15 | {
16 |     int index, argc_n;
17 |     char** argv_n;
18 | 
19 |     Context ctx((uid_t)-1, (gid_t)-1);
20 |     ctx.setDeviceFilePath((char*)USR_DEF_DEV);
21 |     ctx.setStatusFilePath((char*)STATUS_CFG);
22 | 
23 |     char** it;
24 |     for (it = argv; it < argv + argc; it++) {
25 |         for (const auto &cmd : command_list) {
26 |             string arg(it[0]);
27 |             if (arg == cmd.name) {
28 |                 index = (int)(it - argv);
29 |                 argv_n = it;
30 |                 argc_n = argc - index;
31 | 
32 |                 // First parse command line for given command
33 |                 if (!cmd.parse_fn(argc_n, argv_n, &ctx)) {
34 |                   // Print out help
35 |                   help_command(nullptr);
36 |                   return ERR_CODE_PARSE_ERROR;
37 |                 }
38 |                 // Execute the command
39 |                 if (!cmd.func(&ctx)) {
40 |                   help_command(nullptr);
41 |                   return ERR_CODE_FUNC_ERROR;
42 |                 }
43 |                 return EXIT_CODE_SUCCESS;
44 |             }
45 |         }
46 |     }
47 |     help_command(&ctx);
48 |     return 0;
49 | }
50 | 


--------------------------------------------------------------------------------
/ACC-Manager/include/utils.hpp:
--------------------------------------------------------------------------------
 1 | #ifndef _ACC_MANAGER_UTILS
 2 | #define _ACC_MANAGER_UTILS
 3 | 
 4 | #include <sys/capability.h>
 5 | #include <sys/stat.h>
 6 | #include <sys/types.h>
 7 | 
 8 | #include <assert.h>
 9 | #include <stdbool.h>
10 | #include <stddef.h>
11 | #include <fcntl.h>
12 | #include <libgen.h>
13 | #include <limits.h>
14 | 
15 | #include <pwd.h>
16 | #include <grp.h>
17 | 
18 | #include "common.hpp"
19 | #include <regex>
20 | #include <vector>
21 | #include <algorithm>
22 | 
23 | #include <fstream>
24 | #include <sstream>
25 | #include <cctype>
26 | 
27 | typedef string (*parseFn)(string, string &, string);
28 | 
29 | bool caseInSensStringCompare(string , string);
30 | 
31 | string joinRootfsPath(const string &, const string &);
32 | 
33 | string cgroupMount(string, string &, string);
34 | string cgroupRoot(string, string &, string);
35 | 
36 | int parseNamespace(int);
37 | bool enterNamespace(int, int*);
38 | bool setCgroup(int, struct stat, const string &);
39 | 
40 | int makeAncestors(string, mode_t);
41 | mode_t getUmask();
42 | 
43 | string parseProcFile(string , parseFn, string &, string);
44 | string findCgroupPath(pid_t);
45 | 
46 | bool isFileExisting(const char*);
47 | 
48 | string generateDeviceId(Device);
49 | 
50 | #define TRIM_SPACE " \t\n\v"
51 | inline std::string trim(std::string s,const std::string& drop = TRIM_SPACE)
52 | {
53 |     std::string r=s.erase(s.find_last_not_of(drop)+1);
54 |     return r.erase(0,r.find_first_not_of(drop));
55 | }
56 | inline std::string rtrim(std::string s,const std::string& drop = TRIM_SPACE)
57 | {
58 |     return s.erase(s.find_last_not_of(drop)+1);
59 | }
60 | inline std::string ltrim(std::string s,const std::string& drop = TRIM_SPACE)
61 | {
62 |     return s.erase(0,s.find_first_not_of(drop));
63 | }
64 | 
65 | string exec(const char* cmd);
66 | 
67 | #endif
68 | 
69 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | .PHONY:all
 2 | 
 3 | OUT_DIR := out
 4 | 
 5 | RUNTIME_BIN := acc-runtime
 6 | RUNTIME_DIR := $(CURDIR)/ACC-Runtime
 7 | HOOK_BIN := acc-runtime-hook
 8 | HOOK_DIR := $(CURDIR)/ACC-Runtime-Hook
 9 | MANAGER_BIN := acc-manager
10 | MANAGER_DIR := $(CURDIR)/ACC-Manager
11 | 
12 | all: $(RUNTIME_BIN) $(HOOK_BIN) $(MANAGER_BIN)
13 | 
14 | all-cont-ubuntu16.04: $(RUNTIME_BIN)-cont-ubuntu16.04 $(HOOK_BIN)-cont-ubuntu16.04 $(MANAGER_BIN)-cont-ubuntu16.04
15 | 
16 | pre:
17 | 	@if [ ! -d "./$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi
18 | 
19 | $(RUNTIME_BIN): pre
20 | 	make -C $(RUNTIME_DIR)
21 | 	cp $(RUNTIME_DIR)/$(OUT_DIR)/$@ $(OUT_DIR)/$@
22 | 
23 | $(HOOK_BIN): pre
24 | 	make -C $(HOOK_DIR)
25 | 	cp $(HOOK_DIR)/$(OUT_DIR)/$@ $(OUT_DIR)/$@
26 | 
27 | $(MANAGER_BIN): pre
28 | 	make -C $(MANAGER_DIR)
29 | 	cp $(MANAGER_DIR)/$(OUT_DIR)/$@ $(OUT_DIR)/$@
30 | 
31 | $(RUNTIME_BIN)-cont-ubuntu16.04: pre
32 | 	make -C $(RUNTIME_DIR) cont-ubuntu16.04
33 | 	cp $(RUNTIME_DIR)/$(OUT_DIR)/$(RUNTIME_BIN) $(OUT_DIR)/$(RUNTIME_BIN)
34 | 
35 | $(HOOK_BIN)-cont-ubuntu16.04: pre
36 | 	make -C $(HOOK_DIR) cont-ubuntu16.04
37 | 	cp $(HOOK_DIR)/$(OUT_DIR)/$(HOOK_BIN) $(OUT_DIR)/$(HOOK_BIN)
38 | 
39 | $(MANAGER_BIN)-cont-ubuntu16.04: pre
40 | 	make -C $(MANAGER_DIR) cont-ubuntu16.04
41 | 	cp $(MANAGER_DIR)/$(OUT_DIR)/$(MANAGER_BIN) $(OUT_DIR)/$(MANAGER_BIN)
42 | 	cp $(MANAGER_DIR)/$(OUT_DIR)/libprotobuf* $(OUT_DIR)/
43 | 
44 | install:
45 | 	make -C $(RUNTIME_DIR) install
46 | 	make -C $(HOOK_DIR) install
47 | 	make -C $(MANAGER_DIR) install
48 | 	if [ -f "/etc/docker/daemon.json" ]; then mv /etc/docker/daemon.json /etc/docker/daemon.json.bak; fi
49 | 	\cp ./daemon.json /etc/docker/daemon.json
50 | uninstall:
51 | 	rm -f /usr/bin/$(RUNTIME_BIN) /usr/bin/$(HOOK_BIN) /usr/bin/$(MANAGER_BIN)
52 | 
53 | clean:
54 | 	make -C $(RUNTIME_DIR) clean
55 | 	make -C $(HOOK_DIR) clean
56 | 	make -C $(MANAGER_DIR) clean
57 | 	rm -rf $(OUT_DIR)
58 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/configure.cc:
--------------------------------------------------------------------------------
 1 | #include "cli.hpp"
 2 | #include "manager.hpp"
 3 | #include "mounter.hpp"
 4 | 
 5 | // Configure format
 6 | //  : acc-manager configure --pid=<PID> --device=<DEVICES> <ROOTFS>
 7 | bool configure_parse(int argc, char** argv, Context* ctx){
 8 |     for (int i = 0; i < argc; i++) {
 9 |         cmatch cm;
10 |         regex r_pid("--pid=([0-9]*)");  // Parse PID
11 |         regex r_dev("--device=(.*)");   // Parse required device
12 |         regex r_rfs("^--");
13 | 
14 |         // PID arg
15 |         if (regex_match(argv[i], cm, r_pid) && cm.size() > 1) {
16 |             ctx->setPid( atoi(cm[1].str().c_str()) );
17 |         }
18 |         // DEVICE arg
19 |         // Required devices are joined with comma(,) --> divide by comma and put it in required device list
20 |         else if (regex_match(argv[i], cm, r_dev) && cm.size() > 1) {
21 |             string arg = cm[1].str();
22 |             string delimiter = ",";
23 |             size_t pos = 0;
24 |             string token;
25 |             while((pos = arg.find(delimiter)) != string::npos){
26 |                 token = arg.substr(0,pos);
27 |                 if(!token.empty())
28 |                     ctx->addReqDevice(token);
29 |                 arg.erase(0,pos+delimiter.length());
30 |             }
31 |             if(!arg.empty())
32 |                 ctx->addReqDevice(arg);
33 |         }
34 |         // ROOTFS parse
35 |         else if (i == argc - 1 && !regex_match(argv[i], r_rfs)) {
36 |             ctx->setRootFs(string(argv[i]));
37 |         }
38 |     }
39 |     return ctx->validate() && ctx->parseOwner();
40 | }
41 | 
42 | // Configure only calls requestDevice of Manager class
43 | bool configure_command(Context* ctx){
44 |     Manager manager(ctx);
45 |     if (!manager.requestDevices()){
46 |         manager.releaseDevices();
47 |         errx(ERR_CODE_PARSE_ERROR,"requesting devices failed");
48 |         return false;
49 |     }
50 |     return true;
51 | }


--------------------------------------------------------------------------------
/ACC-Manager/Makefile:
--------------------------------------------------------------------------------
 1 | .PHONY:all
 2 | 
 3 | INC_DIR := include
 4 | SRC_DIR := src
 5 | OUT_DIR := out
 6 | BUILD_DIR := build
 7 | TEST_DIR := test
 8 | T_PARSER := t_parser
 9 | TARGET := acc-manager
10 | PROTO_NAME := device
11 | SRCS := $(wildcard src/*)
12 | SRCS := $(notdir $(filter-out src/$(TARGET).cc, $(SRCS)))
13 | OBJS := $(SRCS:.cc=.o)
14 | 
15 | FLAGS := -I $(INC_DIR) -I /usr/local/include -L/usr/local/lib -L$(BUILD_DIR) -lpci -lprotobuf -pthread
16 | 
17 | all: $(clean) $(TARGET)
18 | 
19 | $(TARGET):
20 | 	@if [ ! -d "$(BUILD_DIR)" ]; then mkdir $(BUILD_DIR); fi
21 | 	protoc -I=$(OUT_DIR) --cpp_out=$(BUILD_DIR) $(OUT_DIR)/$(PROTO_NAME).proto
22 | 	mv $(BUILD_DIR)/*.h $(INC_DIR)/
23 | 	mv $(BUILD_DIR)/*.cc $(SRC_DIR)/
24 | 	g++ $(SRC_DIR)/* -D_GNU_SOURCE $(FLAGS) -o $(OUT_DIR)/$@ -std=c++11
25 | 
26 | cont-ubuntu16.04:
27 | 	@if [ ! -d "$(OUT_DIR)" ]; then mkdir $(OUT_DIR); fi
28 | 	docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) -t "acc/manager/ubuntu:16.04" -f Make.Dockerfile .
29 | 	docker run --rm -v $(CURDIR)/$(OUT_DIR):/dist:Z -u $(shell id -u):$(shell id -g) "acc/manager/ubuntu:16.04" /bin/bash -c "cp /ACC-Manager/out/$(TARGET) /dist && cp /usr/local/lib/libprotobuf.so.17 /dist"
30 | 
31 | $(T_PARSER): $(clean) $(OBJS)
32 | 	g++ $(TEST_DIR)/tb_device-detector.cc $(addprefix $(BUILD_DIR)/, $^)  $(FLAGS) -o $(OUT_DIR)/$@ -std=c++11
33 | 
34 | %.o: 
35 | 	g++ -c -o $(BUILD_DIR)/$@ $(SRC_DIR)/$*.cc $(FLAGS) -std=c++11
36 | 
37 | install:
38 | 	cp $(OUT_DIR)/$(TARGET) /usr/bin/$(TARGET)
39 | 	if [ -f "$(OUT_DIR)/libprotobuf.so.17" ]; then cp $(OUT_DIR)/libprotobuf.so.17 /usr/local/lib/libprotobuf.so.17; fi
40 | 	/usr/bin/$(TARGET) list
41 | 
42 | clean:
43 | 	rm -rf $(BUILD_DIR)
44 | 	if [ -f "$(OUT_DIR)/$(TARGET)" ]; then rm $(OUT_DIR)/$(TARGET); fi
45 | 	if [ -f "$(OUT_DIR)/libprotobuf.so.17" ]; then rm $(OUT_DIR)/libprotobuf.so.17; fi
46 | 	if [ -f "$(INC_DIR)/$(PROTO_NAME).pb.h" ]; then rm $(INC_DIR)/$(PROTO_NAME).pb.h; fi
47 | 	if [ -f "$(SRC_DIR)/$(PROTO_NAME).pb.cc" ]; then rm $(SRC_DIR)/$(PROTO_NAME).pb.cc; fi
48 | 	if [ "$(shell docker images -q acc/manager/ubuntu:16.04 2> /dev/null)" != "" ]; then docker rmi "acc/manager/ubuntu:16.04"; fi
49 | 


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"flag"
 5 | 	"fmt"
 6 | 	"log"
 7 | 	"os"
 8 | 	"os/exec"
 9 | 	"path/filepath"
10 | 	"strconv"
11 | 	"strings"
12 | 	"syscall"
13 | )
14 | 
15 | var defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
16 | 
17 | func main() {
18 | 	flag.Parse()
19 | 
20 | 	args := flag.Args()
21 | 	if len(args) == 0 {
22 | 		os.Exit(2)
23 | 	}
24 | 
25 | 	switch args[0] {
26 | 	case "prestart":
27 | 		prestartHook()
28 | 		os.Exit(0)
29 | 	case "poststart":
30 | 		os.Exit(0)
31 | 	case "poststop":
32 | 		poststopHook()
33 | 		os.Exit(0)
34 | 	default:
35 | 		os.Exit(2)
36 | 	}
37 | }
38 | 
39 | // Fetch 'acc-manager' binary path
40 | func getManagerPath() string {
41 | 	dirs := filepath.SplitList(os.Getenv("PATH"))
42 | 	dirs = append(dirs, defaultPATH...)
43 | 	if err := os.Setenv("PATH", strings.Join(dirs, ":")); err != nil {
44 | 		log.Panicln("PATH set error : ", err)
45 | 	}
46 | 	path, err := exec.LookPath("acc-manager")
47 | 	if err != nil {
48 | 		log.Panicln("finding binary acc-manager in", os.Getenv("PATH"), " error : ", err)
49 | 	}
50 | 	return path
51 | }
52 | 
53 | // Prestart Hook
54 | // Call : acc-manager configure --device=<DEVICES> --pid=<PID> <ROOTFS>
55 | func prestartHook() {
56 | 	container := parseContainerConfig()
57 | 
58 | 	args := []string{getManagerPath()}
59 | 	args = append(args, "configure")
60 | 	args = append(args, fmt.Sprintf("--device=%s", getDevices(container.env)))
61 | 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.pid), 10)))
62 | 	args = append(args, container.rootfs)
63 | 
64 | 	fmt.Printf("exec command : %v\n", args)
65 | 	err := syscall.Exec(args[0], args, os.Environ())
66 | 	if err != nil {
67 | 		log.Panicln("exec error : ", err)
68 | 	}
69 | }
70 | 
71 | // Poststop Hook
72 | // Call : acc-manager release --pid=<PID>
73 | func poststopHook() {
74 | 	container := parseContainerConfig()
75 | 
76 | 	args := []string{getManagerPath()}
77 | 	args = append(args, "release")
78 | 	args = append(args, fmt.Sprintf("--pid=%s", strconv.FormatUint(uint64(container.pid), 10)))
79 | 
80 | 	fmt.Printf("exec command : %v\n", args)
81 | 	err := syscall.Exec(args[0], args, os.Environ())
82 | 	if err != nil {
83 | 		log.Panicln("exec error : ", err)
84 | 	}
85 | }
86 | 


--------------------------------------------------------------------------------
/ACC-Runtime-Hook/container_config.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"log"
  6 | 	"os"
  7 | 	"path"
  8 | 	"path/filepath"
  9 | 	"strings"
 10 | )
 11 | 
 12 | type Root struct {
 13 | 	Path string `json:"path"`
 14 | }
 15 | 
 16 | type Process struct {
 17 | 	Env []string `json:"env,omitempty"`
 18 | }
 19 | 
 20 | type Spec struct {
 21 | 	Process *Process `json:"process,omitempty"`
 22 | 	Root    *Root    `json:"root,omitempty"`
 23 | }
 24 | 
 25 | type HookState struct {
 26 | 	Pid        int    `json:"pid,omitempty"`
 27 | 	Bundle     string `json:"bundle"`
 28 | 	BundlePath string `json:"bundlePath"`
 29 | }
 30 | 
 31 | type containerConfig struct {
 32 | 	pid    int
 33 | 	rootfs string
 34 | 	env    map[string]string
 35 | }
 36 | 
 37 | // Parse STDIN to parse container configuration
 38 | func parseContainerConfig() (config containerConfig) {
 39 | 	var hook HookState
 40 | 	in := json.NewDecoder(os.Stdin)
 41 | 	if err := in.Decode(&hook); err != nil {
 42 | 		log.Panicln("decode error: ", err)
 43 | 	}
 44 | 
 45 | 	bundle := hook.Bundle
 46 | 	if len(bundle) == 0 {
 47 | 		bundle = hook.BundlePath
 48 | 	}
 49 | 
 50 | 	spec := parseSpec(path.Join(bundle, "config.json"))
 51 | 
 52 | 	rootfsPath, err := filepath.Abs(spec.Root.Path)
 53 | 	if err != nil {
 54 | 		log.Panicln("rootfs parsing error : ", err)
 55 | 	}
 56 | 
 57 | 	return containerConfig{
 58 | 		pid:    hook.Pid,
 59 | 		rootfs: rootfsPath,
 60 | 		env:    parseEnvMap(spec.Process.Env),
 61 | 	}
 62 | }
 63 | 
 64 | // Get required device list from env. var.s
 65 | func getDevices(env map[string]string) string {
 66 | 	ret_devices := []string{}
 67 | 	// Find Key with ACC_VISIBLE_DEVICES_*
 68 | 	for k, v := range env {
 69 | 		if strings.HasPrefix(k,"ACC_VISIBLE_DEVICES_") {
 70 | 			ret_devices = append(ret_devices, v)
 71 | 		}
 72 |         }
 73 | 	if devices, result := env["ACC_VISIBLE_DEVICES"]; result {
 74 | 		ret_devices = append(ret_devices, devices)
 75 | 	}
 76 | 	return strings.Join(ret_devices[:], ",")
 77 | }
 78 | 
 79 | // Convert env. var.s into map
 80 | func parseEnvMap(env []string) (envMap map[string]string) {
 81 | 	envMap = make(map[string]string)
 82 | 	for _, s := range env {
 83 | 		param := strings.SplitN(s, "=", 2)
 84 | 		if len(param) != 2 {
 85 | 			log.Panicln("environment variable is not valid")
 86 | 		}
 87 | 		envMap[param[0]] = param[1]
 88 | 	}
 89 | 	return
 90 | }
 91 | 
 92 | // Parse container spec from JSON
 93 | func parseSpec(path string) (spec *Spec) {
 94 | 	file, err := os.Open(path)
 95 | 	if err != nil {
 96 | 		log.Panicln("OCI spec open error : ", err)
 97 | 	}
 98 | 	defer file.Close()
 99 | 
100 | 	if err = json.NewDecoder(file).Decode(&spec); err != nil || spec.Process == nil || spec.Root == nil {
101 | 		log.Panicln("OCI spec decode error : ", err)
102 | 	}
103 | 	return
104 | }
105 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Accelerator-Docker
 2 | 
 3 | Accelerator-Docker supports running container with accessibility to a target hardware accelerator (e.g., fpga or gpu).  
 4 | It can manage status of all accelerators in a server.
 5 | Accelerator-Docker supports automatic pass-through of PCIe-connected accelerators (FPGA or GPU) in container. Original docker requires setting for mounting device driver, libraries to access specific hardware in the container. In the Accelerator-Docker, however, it automatically runs a container following the pre-determined settings with simple option. Moreover, it provides common interface to manage heterogeneous accelerators. More details are described below.
 6 | 
 7 | <p align="center">
 8 |  <img src="images/Accelerator-Docker.png" alt="drawing" width="600" height="396"/>
 9 | </p>
10 | 
11 | ## Getting Started
12 | ### Dependencies
13 | - Docker
14 | - Go >= 1.6
15 | - Protocol Buffers (C++) & Protoc
16 | 
17 | For dependencies, refer to [Docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/#set-up-the-repository), [Go](https://golang.org/dl/), [Protocol Buffers](https://github.com/protocolbuffers/protobuf/blob/master/src/README.md) 
18 | 
19 | ### Installation
20 | Install dependencies for build
21 | ```bash
22 | $ sudo apt-get update && \
23 |   sudo apt-get install -y build-essential git pkg-config libseccomp-dev libcap-dev libpci-dev libapparmor-dev libselinux1-dev
24 | ```
25 | Now, clone this repository and make it.
26 | ```bash
27 | $ git clone https://github.com/KAIST-NCL/Accelerator-Docker.git
28 | $ cd Accelerator-Docker
29 | $ make
30 | $ sudo make install
31 | 
32 | $ sudo systemctl restart docker #Restart docker daemon to notify that new runtime is added
33 | ```
34 | 
35 | ### Tested Environment
36 | - Ubuntu 16.04
37 | - Docker 18.06.1-ce
38 | - Go 1.11
39 | - Protocol Buffers 3.6.1
40 | 
41 | ## How to run Accelerator-Docker
42 | First, you need to provide device list you want to access from docker container as protocol buffer text form.  
43 | **/etc/accelerator-docker/device.pbtxt**
44 | ```protobuf
45 | accelerators:[{
46 |     type: "xilinx.fpga/kcu1500",
47 |     devices:[{
48 |         name: "KCU-1500",
49 |         device_file:[
50 |             "/dev/xcldev1",
51 |             "/dev/xdmadev1"
52 |         ],
53 |         library:[
54 |             "/PATH/TO/LIB/libxilinxopencl.so"
55 |         ],
56 |         file:[{
57 |             src: "/PATH/IN/HOST/libxclgemdrv.so",
58 |             dst: "/PATH/IN/CONTAINER/libxclgemdrv.so",
59 |         }],
60 |         env:[{
61 |             key: "XILINX_SDX",
62 |             val: "/SDX"
63 |         }]
64 |     }]
65 | }]
66 | ```
67 | - Files specified in 'library' section are mounted to '/usr/lib' of container, while the files specified in 'file' section are mounted to specific path.
68 | 
69 | ```
70 | $ docker run --runtime acc-runtime -e ACC_VISIBLE_DEVICES=KCU-1500
71 | ```
72 | 
73 | ## Using ACC-Manager
74 | Accelerator-Docker not only provides runtime hook for docker but also provides manager registered in device file.
75 | Accelerator-Manager provides status for all devices and which container uses the hardware.
76 | 
77 | ```
78 | $ acc-manager list
79 | ```
80 | 
81 | 
82 | ## Authors
83 | #### KAIST NCL
84 | * [Sunghyun Kim](https://github.com/cqbqdd11519)
85 | * [Eunju Yang](https://github.com/EunjuYang)
86 | 
87 | ## License
88 | This project is released under the [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause), see [LICENSE](LICENSE) for more information.
89 | 
90 | 


--------------------------------------------------------------------------------
/ACC-Runtime/acc-runtime.patch:
--------------------------------------------------------------------------------
  1 | diff --git a/Makefile b/Makefile
  2 | index 0f26a1c..82fd1c2 100644
  3 | --- a/Makefile
  4 | +++ b/Makefile
  5 | @@ -30,7 +30,7 @@ SHELL := $(shell command -v bash 2>/dev/null)
  6 |  .DEFAULT: runc
  7 |  
  8 |  runc: $(SOURCES)
  9 | -	$(GO) build -buildmode=pie $(EXTRA_FLAGS) -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION} $(EXTRA_LDFLAGS)" -tags "$(BUILDTAGS)" -o runc .
 10 | +	$(GO) build -buildmode=pie $(EXTRA_FLAGS) -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION} $(EXTRA_LDFLAGS)" -tags "$(BUILDTAGS)" -o acc-runtime .
 11 |  
 12 |  all: runc recvtty
 13 |  
 14 | diff --git a/libcontainer/rootfs_linux.go b/libcontainer/rootfs_linux.go
 15 | index 6bd6da7..c7b7049 100644
 16 | --- a/libcontainer/rootfs_linux.go
 17 | +++ b/libcontainer/rootfs_linux.go
 18 | @@ -12,7 +12,7 @@ import (
 19 |  	"path/filepath"
 20 |  	"strings"
 21 |  	"time"
 22 | -
 23 | +	"regexp"
 24 |  	"github.com/cyphar/filepath-securejoin"
 25 |  	"github.com/mrunalp/fileutils"
 26 |  	"github.com/opencontainers/runc/libcontainer/cgroups"
 27 | @@ -86,6 +86,25 @@ func prepareRootfs(pipe io.ReadWriter, iConfig *initConfig) (err error) {
 28 |  		return err
 29 |  	}
 30 |  
 31 | +        //Accelerator-Docker : Temporal way for setting environmental variables!
 32 | +        tmp_path := path.Join(config.Rootfs,".tmp_env")
 33 | +        if _, err := os.Stat(tmp_path); !os.IsNotExist(err) {
 34 | +                buf, _ := ioutil.ReadFile(tmp_path)
 35 | +                data := string(buf)
 36 | +                lines := strings.Split(data,"\n")
 37 | +                for _, line := range lines {
 38 | +                        env := strings.Split(line,"=")
 39 | +                        if(len(env) == 2){
 40 | +                            re := regexp.MustCompile(`\$([^ :/]+)`)
 41 | +                            s := re.ReplaceAllStringFunc(env[1],func(m string) string {
 42 | +                                return os.Getenv(m[1:])
 43 | +                            })
 44 | +                            os.Setenv(env[0],s)
 45 | +                        }
 46 | +                }
 47 | +                os.Remove(tmp_path)
 48 | +        }
 49 | +
 50 |  	// The reason these operations are done here rather than in finalizeRootfs
 51 |  	// is because the console-handling code gets quite sticky if we have to set
 52 |  	// up the console before doing the pivot_root(2). This is because the
 53 | diff --git a/utils.go b/utils.go
 54 | index 5165336..25762a1 100644
 55 | --- a/utils.go
 56 | +++ b/utils.go
 57 | @@ -3,6 +3,7 @@ package main
 58 |  import (
 59 |  	"fmt"
 60 |  	"os"
 61 | +	"os/exec"
 62 |  	"path/filepath"
 63 |  	"strconv"
 64 |  	"strings"
 65 | @@ -54,6 +55,26 @@ func fatal(err error) {
 66 |  	os.Exit(1)
 67 |  }
 68 |  
 69 | +func addACCHook(context *cli.Context, spec *specs.Spec) error {
 70 | +	path, err := exec.LookPath("acc-runtime-hook")
 71 | +	if err != nil {
 72 | +		return err
 73 | +	}
 74 | +	args := []string{path}
 75 | +	if spec.Hooks == nil {
 76 | +		spec.Hooks = &specs.Hooks{}
 77 | +	}
 78 | +	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
 79 | +		Path: path,
 80 | +		Args: append(args, "prestart"),
 81 | +	})
 82 | +	spec.Hooks.Poststop = append(spec.Hooks.Poststop, specs.Hook{
 83 | +		Path: path,
 84 | +		Args: append(args, "poststop"),
 85 | +	})
 86 | +	return nil
 87 | +}
 88 | +
 89 |  // setupSpec performs initial setup based on the cli.Context for the container
 90 |  func setupSpec(context *cli.Context) (*specs.Spec, error) {
 91 |  	bundle := context.String("bundle")
 92 | @@ -66,6 +87,10 @@ func setupSpec(context *cli.Context) (*specs.Spec, error) {
 93 |  	if err != nil {
 94 |  		return nil, err
 95 |  	}
 96 | +
 97 | +	if err = addACCHook(context, spec); err != nil {
 98 | +		return nil, err
 99 | +	}
100 |  	return spec, nil
101 |  }
102 |  
103 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/manager.cc:
--------------------------------------------------------------------------------
  1 | #include "manager.hpp"
  2 | 
  3 | // Manager Class
  4 | // : Handles require/release of devices by container
  5 | // : Context argument has container info (rootfs, required devices)
  6 | Manager::Manager(Context* _ctx) : ctx(_ctx){
  7 |     // Parse accelerator list at instantiation
  8 |     if(!parseAndDetectAccelerators()){
  9 |         errx(ERR_CODE_PARSE_DETECT_DEVICES_FAILED,"detect and parsing devices failed");
 10 |     }
 11 |     updateStatusFile();
 12 | }
 13 | 
 14 | // Provide pass through of all requested devices to container
 15 | bool Manager::requestDevices(){
 16 |     list<string> req_devs = ctx->getReqDevices();
 17 |     for(auto const & it_st : req_devs){
 18 |         bool result_it = false;
 19 |         for(auto & it_acc : acc_list){
 20 |             list<Device> dev_list = it_acc.getDevices();
 21 |             for(auto & it_dev : dev_list){
 22 |                 // if given 'ACC_VISIBLE_DEVICES' equal to device name or id or 'all', provide the device
 23 |                 if(caseInSensStringCompare(it_st,it_dev.getName()) || caseInSensStringCompare(it_st,it_dev.getId()) || caseInSensStringCompare(it_st,string("all"))){
 24 |                     if(it_dev.getStatus() == Device::Status::UNAVAILABLE && it_dev.getPid() != ctx->getPid()){
 25 |                         errx(ERR_CODE_DEVICE_LOCKED,"[%s] device locked by other process",it_st.c_str());
 26 |                         return false;
 27 |                     }
 28 |                     if(it_dev.getStatus() == Device::Status::MISCONFIGURED){
 29 |                         errx(ERR_CODE_DEVICE_MISCONFIGURED,"[%s] device is unable to use as it is misconfigured",it_st.c_str());
 30 |                         return false;
 31 |                     }
 32 |                     result_it = true;
 33 |                     requestDevice(it_dev);
 34 |                 }
 35 |             }
 36 |             if(result_it){
 37 |                 it_acc.setDevices(dev_list);
 38 |                 updateStatusFile();
 39 |             }
 40 |         }
 41 |         if(!result_it){
 42 |             errx(ERR_CODE_DEVICE_NOT_FOUND,"[%s] device not found",it_st.c_str());
 43 |             return false;
 44 |         }
 45 |     }
 46 |     
 47 |     return true;
 48 | }
 49 | 
 50 | // Provide pass through of a specific device to container
 51 | bool Manager::requestDevice(Device & dev){
 52 |     Mounter mounter(ctx);
 53 |     dev.setPid(ctx->getPid());
 54 |     dev.setStatus(Device::Status::UNAVAILABLE);
 55 |     if(!mounter.mountDevice(dev)){
 56 |         releaseDevices();
 57 |         errx(1,"mounting devices failed");
 58 |         return false;
 59 |     }
 60 |     return true;
 61 | }
 62 | 
 63 | // Release every devices held by all dead processes, not only by 'this' process
 64 | bool Manager::releaseDevices(){
 65 |     for(auto & it_acc : acc_list){
 66 |         list<Device> dev_list = it_acc.getDevices();
 67 |         for(auto & it_dev : dev_list){
 68 |             //No permission issue for 'kill' because this function is only called by docker running with root permission
 69 |             if(it_dev.getStatus() == Device::Status::UNAVAILABLE && kill(it_dev.getPid(),0) < 0 ){
 70 |                 it_dev.setStatus(Device::Status::AVAILABLE);
 71 |                 it_dev.setPid(0);
 72 |             }
 73 |         }
 74 |         it_acc.setDevices(dev_list);
 75 |     }
 76 |     updateStatusFile();
 77 |     return true;
 78 | }
 79 | 
 80 | // Update stat.pb file
 81 | bool Manager::updateStatusFile(){
 82 |     auto *stat_list = new device::device_list();
 83 |     for(auto it_acc : acc_list){
 84 |         list<Device> dev_list = it_acc.getDevices();
 85 |         for(auto it_dev : dev_list){
 86 |             device::device *dev = stat_list->add_devices();
 87 |             //only need to store name, status, pid, id
 88 |             dev->set_name(it_dev.getName());
 89 |             dev->set_id(it_dev.getId());
 90 |             dev->set_status(it_dev.getStatus() == Device::Status::MISCONFIGURED ? device::device_Status_MISS : it_dev.getStatus() == Device::Status::AVAILABLE ? device::device_Status::device_Status_IDLE : device::device_Status::device_Status_USED);
 91 |             dev->set_pid(it_dev.getPid());
 92 |         }
 93 |     }
 94 |     
 95 |     fstream output(ctx->getStatusFilePath(), ios::out | ios::trunc | ios::binary);
 96 |     if(!stat_list->SerializeToOstream(&output)){
 97 |         errx(1,"Writing status file failed");
 98 |         return false;
 99 |     }
100 |     chmod(ctx->getStatusFilePath(), 0777);
101 |     return true;
102 | }
103 | 
104 | // Parse device list by calling detector and parser
105 | bool Manager::parseAndDetectAccelerators(){
106 |     DeviceParser parser(ctx->getStatusFilePath(), ctx->getDeviceFilePath());
107 |     DeviceDetector detector;
108 | 
109 |     list<Accelerator> tmp_list;
110 |     parser.parse(&tmp_list);
111 |     detector.detect(&acc_list,tmp_list);
112 |     return true;
113 | }
114 | 
115 | list<Accelerator> Manager::getAcceleratorList(){
116 |     return acc_list;
117 | }


--------------------------------------------------------------------------------
/ACC-Manager/include/common.hpp:
--------------------------------------------------------------------------------
  1 | #ifndef _ACC_MANAGER_COMMON
  2 | #define _ACC_MANAGER_COMMON
  3 | 
  4 | #include <cstdint>
  5 | #include <cstring>
  6 | #include <string>
  7 | #include <list>
  8 | #include <map>
  9 | #include <functional>
 10 | #include <unordered_map>
 11 | #include <array>
 12 | #include <iostream>
 13 | #include <cinttypes>
 14 | #include <sys/stat.h>
 15 | #include <signal.h>
 16 | #include <unistd.h>
 17 | #include <stdio.h>
 18 | #include <err.h>
 19 | #include <stdlib.h>
 20 | 
 21 | #define PATH_LEN_MAX 1024
 22 | 
 23 | #define USR_BIN_DIR               "/usr/bin"
 24 | #define LIB_DIR                   "/lib64"
 25 | #define USR_LIB_DIR               "/usr/lib64"
 26 | #define USR_LIB32_DIR             "/usr/lib32"
 27 | #define USR_LIB32_ALT_DIR         "/usr/lib"
 28 | #if defined(__x86_64__)
 29 |     #define USR_LIB_MULTIARCH_DIR    "/usr/lib/x86_64-linux-gnu"
 30 |     #define USR_LIB32_MULTIARCH_DIR  "/usr/lib/i386-linux-gnu"
 31 |     #if !defined(__NR_execveat)
 32 |         #define __NR_execveat 322
 33 |     #endif
 34 | #elif defined(__powerpc64__)
 35 |     #define USR_LIB_MULTIARCH_DIR    "/usr/lib/powerpc64le-linux-gnu"
 36 |     #define USR_LIB32_MULTIARCH_DIR  "/var/empty"
 37 |     #if !defined(__NR_execveat)
 38 |         #define __NR_execveat 362
 39 |     # endif
 40 | #else
 41 |     # error "unsupported architecture"
 42 | #endif
 43 | 
 44 | #define USR_DEF_DEV     "/etc/accelerator-docker/device.pbtxt"
 45 | #define STATUS_CFG      "/etc/accelerator-docker/stat.pb"
 46 | 
 47 | #define ERR_CODE_NO_ROOTFS 21
 48 | #define ERR_CODE_NO_PROC_DIR 22
 49 | #define ERR_CODE_PID_NOT_VALID 23
 50 | #define ERR_CODE_PROCESS_STILL_RUNNING 24
 51 | 
 52 | using namespace std;
 53 | 
 54 | class Device;
 55 | 
 56 | // Context class for storing container info.
 57 | class Context {
 58 |     public:
 59 |         Context(uid_t,gid_t);
 60 |         bool validate();
 61 |         bool parseOwner();
 62 | 
 63 |         uid_t getUid();
 64 |         gid_t getGid();
 65 |         pid_t getPid();
 66 |         string getRootFs();
 67 |         list<string> getReqDevices();
 68 |         char* getDeviceFilePath();
 69 |         char* getStatusFilePath();
 70 | 
 71 |         void setPid(pid_t);
 72 |         void setRootFs(string);
 73 |         void setDeviceFilePath(char*);
 74 |         void setStatusFilePath(char*);
 75 |         void addReqDevice(string);
 76 |     private:
 77 |         uid_t uid;
 78 |         gid_t gid;
 79 |         pid_t pid;
 80 |         string rootfs;
 81 | 
 82 |         char* device_file_path;
 83 |         char* status_file_path;
 84 | 
 85 |         list<string> req_devices_name;
 86 | };
 87 | 
 88 | // Accelerator container, refer to device.proto
 89 | class Accelerator{
 90 |     public:
 91 |         explicit Accelerator(string);
 92 |         string getType();
 93 |         list<Device>& getDevices();
 94 |         void setDevices(list<Device>);
 95 |         void addDevice(Device);
 96 |     private:
 97 |         string type;
 98 |         list<Device> devices;
 99 | };
100 | 
101 | // Device container, refer to device.proto
102 | class Device{
103 |     public:
104 |         enum Status{
105 |             AVAILABLE,
106 |             UNAVAILABLE,
107 |             MISCONFIGURED
108 |         };
109 |         explicit Device(string);
110 | 
111 |         list<string>& getDeviceFiles();
112 |         list<string>& getLibraries();
113 |         list<array<string,2> >& getFiles();
114 |         list<array<string,2> >& getEnvs();
115 |         Device::Status getStatus();
116 |         string getName();
117 |         string getId();
118 |         pid_t getPid();
119 |         string getPciSlot();
120 |         uint16_t getVendorID();
121 |         uint16_t getDeviceID();
122 |         uint16_t getSubVendorID();
123 |         uint16_t getSubDeviceID();
124 |         void setName(string);
125 |         void setId(string);
126 |         void setType(string);
127 |         void setStatus(Device::Status);
128 |         void setPid(pid_t);
129 |         void setPciSlot(string);
130 |         void addDeviceFile(string);
131 |         void addLibrary(string);
132 |         void addFile(string,string);
133 |         void addEnv(string,string);
134 |     private:
135 |         Device::Status status;
136 |         string name;
137 |         string id;
138 |         pid_t pid;
139 |         list<string> devs;              // device files
140 |         list<string> libs;              // library files
141 |         list<array<string,2>> files;    // files
142 |         list<array<string,2>> envs;     // env vars
143 |         string pciSlot;
144 |         uint16_t vendorID;
145 |         uint16_t deviceID;
146 |         uint16_t subVendorID;
147 |         uint16_t subDeviceID;
148 | };
149 | 
150 | // Class for auto-detect --> not implemented yet
151 | class Driver{
152 |     public:
153 |         Driver(string _name, int _vendorID, int _deviceID, int _subVendorID, int _subDeviceID);
154 |         list<string> getModules();
155 |         void addModuleNode(string mod);
156 | 
157 |         int getVendorID();
158 |         int getDeviceID();
159 |         int getSubVendorID();
160 |         int getSubDeviceID();
161 |     private:
162 |         string name;
163 |         list<string> modules;
164 |         //For Matching Devices
165 |         int vendorID;
166 |         int deviceID;
167 |         int subVendorID;
168 |         int subDeviceID;
169 | };
170 | 
171 | #endif
172 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/common.cc:
--------------------------------------------------------------------------------
  1 | #include "common.hpp"
  2 | #include <iostream>
  3 | Context::Context(uid_t _uid, gid_t _gid) : uid(_uid),gid(_gid),pid(0),rootfs(string("")){}
  4 | 
  5 | // Validate given container information
  6 | bool Context::validate(){
  7 |     char proc_dir[PATH_LEN_MAX];
  8 |     struct stat s_root={};
  9 |     struct stat s_proc={};
 10 |     int n;
 11 | 
 12 |     // Existence check for given rootfs
 13 |     if(stat(rootfs.c_str(),&s_root) < 0 || rootfs.empty() || !S_ISDIR(s_root.st_mode)){
 14 |         errx(ERR_CODE_NO_ROOTFS,"rootfs [%s] does not exist",rootfs.c_str());
 15 |         return false;
 16 |     }
 17 | 
 18 |     // Existence check for given pid
 19 |     // Only used for 'configure' right now, which always run as root --> no permission issue for 'kill'
 20 |     // May be deleted when permission issue occurs (pid existence check is done with /proc/{PID} check)
 21 |     if(pid <= 0 || kill(pid,0) < 0){
 22 |         errx(ERR_CODE_PID_NOT_VALID,"pid [%d] not valid",pid);
 23 |         return false;
 24 |     }
 25 | 
 26 |     // Error handling for snprintf : make /proc/{PID} string
 27 |     if( (n = snprintf(proc_dir,sizeof(proc_dir),"/proc/%d",pid)) <0 || n >= PATH_LEN_MAX){
 28 |         errx(ERR_CODE_NO_PROC_DIR,"proc dir [/proc/%d] does not exist",pid);
 29 |         return false;
 30 |     }
 31 | 
 32 |     // Existence check of /proc/{PID} dir.
 33 |     if( (stat(proc_dir,&s_proc) < 0) || !S_ISDIR(s_proc.st_mode)){
 34 |         errx(ERR_CODE_NO_PROC_DIR,"proc dir [/proc/%d] does not exist",pid);
 35 |         return false;
 36 |     }
 37 |     return true;
 38 | }
 39 | 
 40 | // Parse UID,GID of given container
 41 | bool Context::parseOwner(){
 42 |     char proc_dir[PATH_LEN_MAX];
 43 |     struct stat s_proc ={};
 44 |     int n;
 45 |     if( (n = snprintf(proc_dir,sizeof(proc_dir),"/proc/%d",pid)) < 0 || n >= PATH_LEN_MAX ){
 46 |         errx(ERR_CODE_NO_PROC_DIR,"proc dir [/proc/%d] does not exist",pid);
 47 |         return false;
 48 |     }
 49 |     if( (stat(proc_dir,&s_proc) < 0) || !S_ISDIR(s_proc.st_mode)){
 50 |         errx(ERR_CODE_NO_PROC_DIR,"proc dir [/proc/%d] does not exist",pid);
 51 |         return false;
 52 |     }
 53 |     uid = s_proc.st_uid;
 54 |     gid = s_proc.st_gid;
 55 |     return true;
 56 | }
 57 | 
 58 | uid_t Context::getUid(){ return uid; }
 59 | gid_t Context::getGid(){ return gid; }
 60 | pid_t Context::getPid(){ return pid; }
 61 | string Context::getRootFs(){ return rootfs; }
 62 | list<string> Context::getReqDevices(){ return req_devices_name; }
 63 | char* Context::getDeviceFilePath(){ return device_file_path; }
 64 | char* Context::getStatusFilePath(){ return status_file_path; }
 65 | 
 66 | void Context::setPid(const pid_t _pid){pid=_pid;}
 67 | void Context::setRootFs(const string _rootfs){rootfs=_rootfs;}
 68 | void Context::setDeviceFilePath(char* path){device_file_path=path;}
 69 | void Context::setStatusFilePath(char* path){status_file_path=path;}
 70 | void Context::addReqDevice(const string dev){req_devices_name.push_back(dev);}
 71 | 
 72 | 
 73 | Accelerator::Accelerator(const string _type): type(_type){}
 74 | string Accelerator::getType(){ return type; }
 75 | list<Device>& Accelerator::getDevices(){ return devices; }
 76 | void Accelerator::setDevices(const list<Device> _devs){
 77 |     devices = _devs;
 78 | }
 79 | void Accelerator::addDevice(const Device _device){
 80 |     devices.push_back(_device);
 81 | }
 82 | 
 83 | 
 84 | Device::Device(const string _name): name(_name){
 85 |     status = Status::AVAILABLE;
 86 |     pid = 0;
 87 | }
 88 | list<string>& Device::getDeviceFiles(){ return devs; }
 89 | list<string>& Device::getLibraries(){ return libs; }
 90 | list<array<string,2>>& Device::getFiles(){ return files; }
 91 | list<array<string,2>>& Device::getEnvs(){ return envs; }
 92 | Device::Status Device::getStatus(){ return status; }
 93 | string Device::getName(){ return name; }
 94 | string Device::getId(){ return id; }
 95 | pid_t Device::getPid(){ return pid; }
 96 | string Device::getPciSlot(){ return pciSlot; }
 97 | uint16_t Device::getVendorID(){ return vendorID; }
 98 | uint16_t Device::getDeviceID(){ return deviceID; }
 99 | uint16_t Device::getSubVendorID(){ return subVendorID; }
100 | uint16_t Device::getSubDeviceID(){ return subDeviceID; }
101 | 
102 | void Device::addDeviceFile(const string path){
103 |     devs.push_back(path);
104 | }
105 | void Device::addLibrary(const string lib){
106 |     libs.push_back(lib);
107 | }
108 | void Device::addFile(const string src, const string dst){
109 |     files.push_back({src, dst});
110 | }
111 | void Device::addEnv(const string key, const string val){
112 |     envs.push_back({key, val});
113 | }
114 | void Device::setStatus(const Device::Status stat){
115 |     status = stat;
116 | }
117 | void Device::setName(const string _name){
118 |     name = _name;
119 | }
120 | void Device::setId(const string _id){
121 |     id = _id;
122 | }
123 | void Device::setPid(const pid_t _pid){
124 |     pid = _pid;
125 | }
126 | void Device::setPciSlot(const string slot){
127 |     pciSlot = slot;
128 | }
129 | 
130 | Driver::Driver(const string _name, const int _vendorID, const int _deviceID, const int _subVendorID, const int _subDeviceID)
131 |  : name(_name), vendorID(_vendorID), deviceID(_deviceID), subVendorID(_subVendorID), subDeviceID(_subDeviceID){}
132 | list<string> Driver::getModules(){ return modules; }
133 | int Driver::getVendorID(){ return vendorID; }
134 | int Driver::getDeviceID(){ return deviceID; }
135 | int Driver::getSubVendorID(){ return subVendorID; }
136 | int Driver::getSubDeviceID(){ return subDeviceID; }
137 | void Driver::addModuleNode(string mod){
138 |     modules.push_back(mod);
139 | }


--------------------------------------------------------------------------------
/ACC-Manager/src/utils.cc:
--------------------------------------------------------------------------------
  1 | #include "utils.hpp"
  2 | 
  3 | bool caseInSensStringCompare(string str1, string str2){
  4 | 	return ((str1.size() == str2.size()) && equal(str1.begin(), str1.end(), str2.begin(), [](char & c1, char & c2){
  5 |         return (c1 == c2 || toupper(c1) == toupper(c2));
  6 |     }));
  7 | }
  8 | 
  9 | string joinRootfsPath(const string & rootfs, const string & path){
 10 |     stringstream ss;
 11 |     string result = rootfs;
 12 |     while(result.c_str()[result.size()-1] == '/'){
 13 |         result = result.substr(0,result.size()-1);
 14 |     }
 15 |     ss << result << path;
 16 |     return ss.str();
 17 | }
 18 | 
 19 | bool setCgroup(int pid, struct stat s, const string & cg_path){
 20 |     char buffer[PATH_LEN_MAX];
 21 |     snprintf(buffer, sizeof(buffer), "c %u:%u rw", major(s.st_rdev), minor(s.st_rdev));
 22 | 
 23 |     ofstream outFile;
 24 |     outFile.open(cg_path+"/devices.allow",ios::app);
 25 |     if(outFile.is_open()){
 26 |         outFile << buffer;
 27 |     } else
 28 |         return false;
 29 |     return true;
 30 | }
 31 | 
 32 | string findCgroupPath(pid_t pid){
 33 |     string root_prefix;
 34 |     string mount, root;
 35 |     string path;
 36 | 
 37 |     path = "/proc/" + to_string(getppid()) + "/mountinfo";
 38 |     mount = parseProcFile(path, cgroupMount, root_prefix, "devices");
 39 |     path = "/proc/" + to_string(pid) + "/cgroup";
 40 |     root = parseProcFile(path, cgroupRoot, root_prefix, "devices");
 41 | 
 42 |     return mount+root;
 43 | }
 44 | 
 45 | string parseProcFile(string procf, parseFn parse, string & prefix, string subsys){
 46 |     ifstream inFile(procf);
 47 |     string path;
 48 |     string line;
 49 |     if(inFile.is_open()){
 50 |         while(getline(inFile,line)){
 51 |             replace(line.begin(),line.end(),'\n','\0');
 52 |             if(line.empty())
 53 |                 continue;
 54 |             if(!(path = parse(line,prefix,subsys)).empty())
 55 |                 break;
 56 |         }
 57 |     }
 58 | 
 59 |     return path;
 60 | }
 61 | 
 62 | int parseNamespace(int pid){
 63 |     char ns_path[PATH_LEN_MAX];
 64 |     int fd;
 65 |     if (pid == 0){
 66 |         snprintf(ns_path, sizeof(ns_path), "/proc/self/ns/mnt");
 67 |     }else{
 68 |         snprintf(ns_path, sizeof(ns_path), "/proc/%d/ns/mnt", pid);
 69 |     }
 70 |     fd = open(ns_path, O_RDONLY | O_CLOEXEC);
 71 |     return fd;
 72 | }
 73 | 
 74 | bool enterNamespace(int pid, int *_fd){
 75 |     int fd;
 76 |     if (_fd == nullptr){
 77 |         fd = parseNamespace(pid);
 78 |     }else{
 79 |         fd = *_fd;
 80 |     }
 81 |     setns(fd, CLONE_NEWNS);
 82 |     close(fd);
 83 |     return true;
 84 | }
 85 | 
 86 | int makeAncestors(string path, mode_t perm){
 87 |     struct stat s = {};
 88 | 
 89 |     if(path.empty() || path[0] == '.')
 90 |         return (0);
 91 | 
 92 |     if (stat(path.c_str(), &s) == 0){
 93 |         if (S_ISDIR(s.st_mode))
 94 |             return (0);
 95 |         errno = ENOTDIR;
 96 |     }
 97 |     if (errno != ENOENT)
 98 |         return (-1);
 99 | 
100 |     size_t pos = path.find_last_of('/');
101 |     if(pos != string::npos){
102 |         string parent = path.substr(0,pos);
103 |         if(makeAncestors(parent,perm) < 0)
104 |             return -1;
105 |     }
106 | 
107 |     return mkdir(path.c_str(), perm);
108 | }
109 | 
110 | mode_t getUmask(){
111 |     mode_t mask;
112 | 
113 |     mask = umask(0);
114 |     umask(mask);
115 |     return (mask);
116 | }
117 | 
118 | string cgroupMount(string line, string & prefix, string subsys){
119 |     string root, mount, fstype, substr;
120 | 
121 |     size_t pos_dash = line.find('-');
122 |     if(pos_dash == string::npos)
123 |         return "";
124 |     string first = line.substr(0,pos_dash-1);
125 |     string second = line.substr(pos_dash);
126 |     cmatch cm;
127 | 
128 |     regex r_first("[^ ]+ [^ ]+ [^ ]+ ([^ ]+) ([^ ]+) .*");
129 |     if (regex_match(first.c_str(), cm, r_first) && cm.size() > 2) {
130 |         root = cm[1];
131 |         mount = cm[2];
132 |     }else{
133 |         return "";
134 |     }
135 | 
136 |     regex r_second("[^ ]* ([^ ]+) [^ ]+ (.*)");
137 |     if (regex_match(second.c_str(), cm, r_second) && cm.size() > 2) {
138 |         fstype = cm[1];
139 |         substr = cm[2];
140 |     }else{
141 |         return "";
142 |     }
143 | 
144 |     if(root.empty() || mount.empty() || fstype.empty() || substr.empty())
145 |         return "";
146 |     if(fstype != "cgroup")
147 |         return "";
148 |     if(substr.find(subsys) == string::npos)
149 |         return "";
150 |     if(root.size() >= PATH_LEN_MAX || root.find("/..") == 0)
151 |         return "";
152 | 
153 |     prefix = root;
154 |     return mount;
155 | }
156 | 
157 | string cgroupRoot(string line, string & prefix, string subsys){
158 |     string root, substr;
159 |     size_t pos;
160 | 
161 |     cmatch cm;
162 |     regex reg("[^:]+:([^:]+):([^:]+)");
163 |     if (regex_match(line.c_str(), cm, reg) && cm.size() > 2) {
164 |         substr = cm[1];
165 |         root = cm[2];
166 |     }else{
167 |         return "";
168 |     }
169 | 
170 |     if(substr.empty() || root.empty())
171 |         return "";
172 |     if(substr.find(subsys) == string::npos)
173 |         return "";
174 |     if(root.size() >= PATH_LEN_MAX || root.find("/..") == 0)
175 |         return "";
176 |     if(prefix != "/" && (pos = root.find(prefix)) == 0){
177 |         root = root.substr(pos);
178 |     }
179 |     return root;
180 | }
181 | 
182 | bool isFileExisting(const char* path){
183 |     struct stat s = {};
184 |     return (stat (path, &s) == 0); 
185 | }
186 | 
187 | unsigned hash_str(string s){
188 |    unsigned h = 37;
189 |    for(const char & c : s) {
190 |      h = (h * 54059) ^ (c * 76963);
191 |    }
192 |    return h % 86969;
193 | }
194 | 
195 | //Generate device id by hashing device/library file list --> if device/library file list changes, id changes
196 | string generateDeviceId(Device dev){
197 |     unsigned hash = 0;
198 |     for(const string & path: dev.getDeviceFiles()){
199 |         hash += hash_str(path);
200 |     }
201 |     for(const string & path: dev.getLibraries()){
202 |         hash += hash_str(path);
203 |     }
204 |     return to_string(hash);
205 | }
206 | 
207 | string exec(const char* cmd) {
208 |     array<char, 128> buffer;
209 |     string result;
210 |     unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd, "r"), pclose);
211 |     if (!pipe) {
212 |         throw runtime_error("popen() failed!");
213 |     }
214 |     while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
215 |         result += buffer.data();
216 |     }
217 |     return result;
218 | }
219 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/mounter.cc:
--------------------------------------------------------------------------------
  1 | #include "mounter.hpp"
  2 | #include <iostream>
  3 | #include <fstream>
  4 | 
  5 | // Mounter Class
  6 | // : Mount files needed for accelerators
  7 | // : Context has information of container (rootfs, UID, GID)
  8 | Mounter::Mounter(Context* _cont) : cont(_cont) {}
  9 | 
 10 | // Mount files for all devices contained in given list
 11 | bool Mounter::mountDevices(list<Device> devices){
 12 |     bool result = true;
 13 |     for (auto const & it : devices){
 14 |         result = result && mountDevice(it);
 15 |     }
 16 |     return result;
 17 | }
 18 | 
 19 | // Mount files for a specific device
 20 | bool Mounter::mountDevice(Device device){
 21 |     bool result = true;
 22 |     list<string> dev_files = device.getDeviceFiles();
 23 |     list<string> lib_files = device.getLibraries();
 24 |     list<array<string,2>> files = device.getFiles();
 25 |     list<array<string,2>> envs = device.getEnvs();
 26 | 
 27 |     int fd_mnt_ori;
 28 |     fd_mnt_ori = parseNamespace(0); // Original namespace of acc-manager process
 29 |     enterNamespace(cont->getPid(), nullptr); // Enter container's namespace
 30 |     result = result &&
 31 |             mountDeviceFiles(dev_files) &&
 32 |             mountLibraries(lib_files) &&
 33 |             mountFiles(files) &&
 34 |             setEnvs(envs);
 35 |     enterNamespace(0, &fd_mnt_ori); // Escape from container's namespace
 36 |     return result;
 37 | }
 38 | 
 39 | // Mount all device files(/dev/<FILE>) on rootfs
 40 | bool Mounter::mountDeviceFiles(list<string> dev_files){
 41 |     string cg_path;
 42 |     bool result = true;
 43 | 
 44 |     // cgroup path
 45 |     cg_path = findCgroupPath(cont->getPid());
 46 |     for (auto const & it : dev_files){
 47 |         result = result && mountDeviceFile(it,cg_path);
 48 |     }
 49 |     return result;
 50 | }
 51 | 
 52 | // Mount a device file on rootfs
 53 | bool Mounter::mountDeviceFile(string dev, string cg_path){
 54 |     char src[PATH_LEN_MAX];
 55 |     char dst[PATH_LEN_MAX];
 56 |     struct stat s = {};
 57 | 
 58 |     strcpy(src, dev.c_str());
 59 |     stat(src, &s);
 60 | 
 61 |     strcpy(dst, cont->getRootFs().c_str());
 62 |     strcat(dst, src);
 63 | 
 64 |     createDev(dst, s); // Create device file on container
 65 |     setCgroup(cont->getPid(), s,cg_path); // Allow the namespace can use accelerator file
 66 |     return true;
 67 | }
 68 | 
 69 | // Create device file
 70 | bool Mounter::createDev(string dst, struct stat s){
 71 |     mode_t mode, perm;
 72 |     char *p;
 73 | 
 74 |     mode = s.st_mode;
 75 | 
 76 |     p = strdup(dst.c_str());
 77 |     perm = (0777 & ~getUmask()) | S_IWUSR | S_IXUSR;
 78 |     makeAncestors(dirname(p), perm);
 79 |     perm = 0644 & ~getUmask() & mode;
 80 |     mknod(dst.c_str(), perm | S_IFCHR, s.st_rdev);
 81 |     return true;
 82 | }
 83 | 
 84 | // Mount all library files given in the list
 85 | bool Mounter::mountLibraries(list<string> lib_files){
 86 |     bool result = true;
 87 |     for (auto const & it : lib_files){
 88 |         result = result && mountLibrary(it);
 89 |     }
 90 |     return result;
 91 | }
 92 | 
 93 | // Mount a specific library file
 94 | bool Mounter::mountLibrary(string lib){
 95 |     string base = lib.substr(lib.find_last_of("/\\") + 1);
 96 |     string dst = "/usr/lib/"+base;
 97 |     return mountFile(lib,dst);
 98 | }
 99 | 
100 | // Mount all files given in the list
101 | bool Mounter::mountFiles(list<array<string,2>> files){
102 |     bool result = true;
103 |     for (auto const & it : files){
104 |         result = result && mountFile(it.at(0),it.at(1));
105 |     }
106 |     return result;
107 | }
108 | 
109 | // Mount a file from src to dst_rel
110 | bool Mounter::mountFile(string src,string dst_rel){
111 |     string dst = joinRootfsPath(cont->getRootFs(),dst_rel);   // dst : {ROOTFS}/{DST_REL}
112 |     const char* src_c = src.c_str();
113 |     const char* dst_c = dst.c_str();
114 |     char dir_tmp[PATH_LEN_MAX];
115 |     struct stat mode = {};
116 |     mode_t perm;
117 |     
118 |     if(src.empty() || dst.empty())
119 |         return false;
120 |     if(!src.empty() && src_c[0] != '/' ){
121 |         //TODO: handle relative path
122 |         errx(ERR_CODE_ONLY_ABS_PATH_ALLOWED,"library [%s] should be in absolute path",src_c);
123 |         return false;
124 |     }
125 |     lstat(src_c, &mode);
126 |     strcpy(dir_tmp,dst_c);
127 |     perm = (0777 & ~getUmask()) | S_IWUSR | S_IXUSR;
128 |     makeAncestors(dirname(dir_tmp), perm);
129 | 
130 |     if(S_ISLNK(mode.st_mode)){ 
131 |         // If it is a link, make a link as same way from host and mount target file (recursively)
132 |         char lnk_buf[1024];
133 |         ssize_t len;
134 |         if( (len = readlink(src_c,lnk_buf,sizeof(lnk_buf)-1)) < 0){
135 |             return false;
136 |         }
137 |         lnk_buf[len] = '\0';
138 |         string lnk_src;
139 |         string lnk_dst;
140 |         // Linked in absolute path
141 |         if(lnk_buf[0] == '/'){
142 |             lnk_dst = joinRootfsPath(cont->getRootFs(),string(lnk_buf));
143 |             lnk_src = string(lnk_buf);
144 |         }
145 |         // Linked in relative path
146 |         else{
147 |             //Tokenize dst_c and append lnk_buf
148 |             const char delimeter[2] = "/";
149 |             char* token;
150 |             char* tmp_tok;
151 | 
152 |             string tmp_s = dst_rel;
153 |             char* dst_tmp = (char*)tmp_s.c_str();
154 |             lnk_dst = string("");
155 |             token = strtok(dst_tmp,delimeter);
156 |             while(token != nullptr){
157 |                 tmp_tok = strtok(nullptr,delimeter);
158 |                 if(tmp_tok != nullptr)
159 |                     lnk_dst += string("/") + string(token);
160 |                 token = tmp_tok;
161 |             }
162 |             lnk_dst += string("/") + string(lnk_buf);
163 | 
164 |             tmp_s = src;
165 |             char* src_tmp = (char*)tmp_s.c_str();
166 |             lnk_src = string("");
167 |             token = strtok(src_tmp,delimeter);
168 |             while(token != nullptr){
169 |                 tmp_tok = strtok(nullptr,delimeter);
170 |                 if(tmp_tok != nullptr)
171 |                     lnk_src += string("/") + string(token);
172 |                 token = tmp_tok;
173 |             }
174 |             lnk_src = lnk_src + string("/") + string(lnk_buf);
175 |         }
176 |         symlink(lnk_buf,dst_c); // Make link in container
177 |         return mountFile(lnk_src,lnk_dst);
178 |     }else if(S_ISREG(mode.st_mode) || S_ISDIR(mode.st_mode)){
179 |         if(S_ISREG(mode.st_mode)){
180 |             std::ofstream dest(dst_c, std::ios::binary);
181 |             dest.close();
182 |         }
183 |         else{
184 |             mkdir(dst_c,perm);
185 |         }
186 |         mount(src_c,dst_c,NULL,MS_BIND,NULL);
187 |         mount(NULL,dst_c,NULL,MS_BIND | MS_REMOUNT | MS_RDONLY | MS_NODEV | MS_NOSUID | MS_REC,NULL);
188 |         return true;
189 |     }else{
190 |         return false;
191 |     }
192 |     return false;
193 | }
194 | 
195 | // Set env. var
196 | // Temporal naive way (storing /.tmp_env file from acc-manager and parse it at acc-runtime)
197 | bool Mounter::setEnvs(list<array<string,2>> envs){
198 |     string dst = joinRootfsPath(cont->getRootFs(),"/.tmp_env");
199 | 
200 |     ofstream outFile;
201 |     outFile.open(dst, fstream::out | fstream::app);
202 |     if(outFile.is_open()){
203 |         for(auto const & it : envs){
204 |             outFile << it.at(0) << "=" << it.at(1) << endl;
205 |         }
206 |         outFile.close();
207 |     }
208 |     return true;
209 | }
210 | 
211 | 


--------------------------------------------------------------------------------
/ACC-Manager/src/device-parser.cc:
--------------------------------------------------------------------------------
  1 | #include "device-parser.hpp"
  2 | 
  3 | using namespace std;
  4 | using namespace google;
  5 | 
  6 | // Device Parser
  7 | // : Parse user-specified devices in /etc/accelerator-docker/{device.pbtxt,stat.pb}
  8 | DeviceParser::DeviceParser(char* _statusFilePath, char* _deviceFilePath) : statusFilePath(_statusFilePath), deviceFilePath(_deviceFilePath){}
  9 | 
 10 | // Parse device list from /etc/accelerator-docker/device.pbtxt and move it to 'to'
 11 | bool DeviceParser::parse(list<Accelerator>* to){
 12 |     auto *usr_list = new device::accelerator_list();
 13 |     auto *stat_list = new device::device_list();
 14 | 
 15 |     string statusFileParent = string(statusFilePath).substr(0, string(statusFilePath).find_last_of("/\\"));
 16 |     string deviceFileParent = string(deviceFilePath).substr(0, string(deviceFilePath).find_last_of("/\\"));
 17 | 
 18 |     makeAncestors(statusFileParent, 0777);
 19 |     makeAncestors(deviceFileParent, 0777);
 20 | 
 21 |     // Status file : stat.pb (binary)
 22 |     fstream fd1(statusFilePath,ios::in | ios::binary);
 23 |     stat_list->ParseFromIstream(&fd1);
 24 |     map<string,Device> devs_stat = devListToDevMap(protoToDeviceList(stat_list));
 25 | 
 26 |     // User-specified devices list : device.pbtxt (text form)
 27 |     int fd0 = open(deviceFilePath, O_RDONLY);
 28 |     protobuf::io::FileInputStream fileInput(fd0);
 29 |     protobuf::TextFormat::Parse(&fileInput, usr_list);
 30 |     close(fd0);
 31 |     list<Accelerator> accs_usr = protoToAccelerator(usr_list);
 32 | 
 33 |     //Status file + User defined device file --> From status file, parse status and pid of only those who are defined in user defined list
 34 |     //if no id set to some device, set it here
 35 |     for(auto & it_acc : accs_usr){
 36 |         list<Device> devs = it_acc.getDevices();
 37 |         // For each device instance of an accelerator type, find if status is saved in stat.pb
 38 |         for(auto & it_dev : devs){
 39 |             string dev_id = generateDeviceId(it_dev);
 40 |             it_dev.setId(dev_id);
 41 |             auto dev_stat = devs_stat.find(dev_id);
 42 |             if(dev_stat != devs_stat.end()){
 43 |                 it_dev.setId(dev_stat->second.getId());
 44 |                 it_dev.setStatus(dev_stat->second.getStatus());
 45 |                 it_dev.setPid(dev_stat->second.getPid());
 46 |                 // If pid is unavailable(not running), it is considered as available
 47 |                 struct stat s = {};
 48 |                 string proc_str = "/proc/"+to_string(it_dev.getPid());
 49 |                 const char* proc_str_c = proc_str.c_str();
 50 |                 stat(proc_str_c, &s);
 51 |                 if(it_dev.getStatus() == Device::Status::UNAVAILABLE && !S_ISDIR(s.st_mode)){
 52 |                     it_dev.setStatus(Device::Status::AVAILABLE);
 53 |                     it_dev.setPid(0);
 54 |                 }
 55 |             }
 56 |         }
 57 |         it_acc.setDevices(devs);
 58 |     }
 59 | 
 60 |     *to = accs_usr;
 61 |     return isListValid(*to);
 62 | }
 63 | 
 64 | // From protobuf type 'accelerator_list' to list of Accelerator class
 65 | list<Accelerator> DeviceParser::protoToAccelerator(device::accelerator_list* proto_acc){
 66 |     list<Accelerator> accs;
 67 |     for(int idx = 0; idx < proto_acc->accelerators_size(); idx++){
 68 |         const device::accelerator & t_acc = proto_acc->accelerators(idx);
 69 |         Accelerator acc(t_acc.type());
 70 |         list<Device> devs;
 71 |         for(int i = 0; i < t_acc.devices_size() ; i ++){
 72 |             const device::device & t_dev = t_acc.devices(i);
 73 |             devs.push_back(protoToDevice(&t_dev));
 74 |         }
 75 |         acc.setDevices(devs);
 76 |         accs.push_back(acc);
 77 |     }
 78 |     return accs;
 79 | }
 80 | 
 81 | // From protobuf type 'device_list' to list of Device class
 82 | list<Device> DeviceParser::protoToDeviceList(const device::device_list* proto_dev){
 83 |     list<Device> devs;
 84 |     // Devie class inherits device class defined by protocolbuffer
 85 |     for(int idx = 0; idx < proto_dev->devices_size(); idx ++){
 86 |         const device::device & t_dev = proto_dev->devices(idx);
 87 |         Device dev = protoToDevice(&t_dev);
 88 |         devs.push_back(dev);
 89 |     }
 90 |     return devs;
 91 | }
 92 | 
 93 | // From protobuf type 'device_list' to list of Device class
 94 | Device DeviceParser::protoToDevice(const device::device* t_dev){
 95 |     Device dev(t_dev->name());
 96 |     if(t_dev->has_pci())
 97 |         dev.setPciSlot(t_dev->pci());
 98 |     if(t_dev->has_status())
 99 |         dev.setStatus(t_dev->status() == device::device_Status::device_Status_IDLE ? Device::Status::AVAILABLE : Device::Status::UNAVAILABLE);
100 |     if(t_dev->has_pid())
101 |         dev.setPid(t_dev->pid());
102 |     if(t_dev->has_id())
103 |         dev.setId(t_dev->id());
104 | 
105 |     // Device driver file path
106 |     for(int i = 0; i < t_dev->device_file_size(); i++)
107 |         dev.addDeviceFile(t_dev->device_file(i));
108 | 
109 |     // Library path
110 |     for(int i = 0; i < t_dev->library_size(); i++)
111 |         dev.addLibrary(t_dev->library(i));
112 | 
113 |     // File path
114 |     for(int i = 0; i < t_dev->file_size(); i++)
115 |         dev.addFile(t_dev->file(i).src(),t_dev->file(i).dst());
116 | 
117 |     // Env var.
118 |     for(int i = 0; i < t_dev->env_size(); i++)
119 |         dev.addEnv(t_dev->env(i).key(),t_dev->env(i).val());
120 |     return dev;
121 | }
122 | 
123 | // Convert from list to map easy to find by id
124 | map<string,Device> DeviceParser::devListToDevMap(list<Device> devList){
125 |     map<string,Device> devMap;
126 | 
127 |     for(auto it : devList){
128 |         devMap.insert(pair<string,Device>(it.getId(),it));
129 |     }
130 | 
131 |     return devMap;
132 | }
133 | 
134 | // Check if accelerator list is valid
135 | bool DeviceParser::isListValid(list<Accelerator>& accList){
136 |     bool res = true;
137 |     deviceNamesTmp.clear();
138 |     for(auto & it : accList){
139 |         res = isAcceleratorValid(it) && res;
140 |     }
141 |     return res;
142 | }
143 | 
144 | // Check if an Accelerator class valid
145 | bool DeviceParser::isAcceleratorValid(Accelerator& acc){
146 |     bool accValid = true;
147 |     // Accelerator type check (for k8s compatibility)
148 |     regex reg("([0-9a-zA-Z-\\._]{0,253}\\/)?[0-9a-zA-Z]?[0-9a-zA-Z-\\._]{0,61}[0-9a-zA-Z]?");
149 |     if(! regex_match(acc.getType(), reg)){
150 |         cerr << "ERROR [" << acc.getType() << "] Type is not valid. Please use k8s 'label' naming rule.\n";
151 |         accValid = false;
152 |     }
153 | 
154 |     list<Device>& deviceList = acc.getDevices();
155 |     bool res = false;
156 |     for(auto & it : deviceList){
157 |         bool tmp = isDeviceValid(it);
158 |         // Validity check and set as MISCONFIGURED status if something is not configured correctly
159 |         if(!tmp || !accValid)
160 |             it.setStatus(Device::Status::MISCONFIGURED);
161 |         res = tmp && res && accValid;
162 |     }
163 |     return res;
164 | }
165 | 
166 | // Check if a Device class valid
167 | bool DeviceParser::isDeviceValid(Device& device){
168 |     bool res = true;
169 |     list<string>& devs = device.getDeviceFiles();
170 |     list<string>& libs = device.getLibraries();
171 |     list<array<string,2> >& files = device.getFiles();
172 | 
173 |     // Check naming rule
174 |     device.setName(trim(device.getName()));
175 |     regex reg("([0-9a-zA-Z-\\._]{0,253}\\/)?[0-9a-zA-Z]?[0-9a-zA-Z-\\._]{0,61}[0-9a-zA-Z]?");
176 |     if(! regex_match(device.getName(), reg)){
177 |         cerr << "ERROR [" << device.getName() << "] Name is not valid. Please use k8s 'label' naming rule.\n";
178 |         res = false;
179 |     }
180 | 
181 |     // Check name duplications
182 |     if(deviceNamesTmp.find(device.getName()) == deviceNamesTmp.end()){
183 |         deviceNamesTmp.insert(device.getName());
184 |     }else{
185 |         cerr << "ERROR [" << device.getName() << "] Name is duplicated.\n";
186 |         res = false;
187 |     }
188 | 
189 |     // Convert shared library to absolute path using ldcache
190 |     for(auto &it : libs){
191 |         if(it.find_first_of('/') !=  0){
192 |             if(ld_cache.empty()){
193 |                 ld_cache = exec("ldconfig -p");
194 |             }
195 |             if(machine_arch.empty()){
196 |                 struct utsname sysinfo;
197 |                 uname(&sysinfo);
198 |                 machine_arch=string(sysinfo.machine);
199 |             }
200 |             string libtype;
201 |             string abs_path;
202 | 
203 |             regex specialChars { R"([-[\]{}()*+?.,\^$|#\s])" };
204 |             string sanitized = regex_replace( it, specialChars, R"(\$&)" );
205 | 
206 |             regex reg("[ \t\n]*("+sanitized+") \\(([^\\)]*)\\) => ([^ \t\n]*)[ \t\n]*", regex::optimize);
207 |             smatch m;
208 | 
209 |             istringstream f(ld_cache);
210 |             string line;
211 |             while(getline(f, line)){
212 |                 if(regex_match(line, m, reg)){
213 |                     string now_libtype = m.str(2);
214 |                     string now_abs_path = m.str(3);
215 | 
216 |                     if((libtype.empty() && abs_path.empty()) || (libtype.find(machine_arch) == string::npos && now_libtype.find(machine_arch) != string::npos)){
217 |                         libtype = now_libtype;
218 |                         abs_path = now_abs_path;
219 |                     }
220 |                 }
221 |             }
222 |             if(!libtype.empty() && !abs_path.empty()){
223 |                 if(ACC_VERBOSE){
224 |                     cout << "WARNING [" << device.getName() << "] Library [" << it << "] changed to [" << abs_path << "] by using LD Cache.\n"
225 |                             "         This makes starting container slow. If possible, specify library path in absolute path.\n";
226 |                 }
227 |                 it = abs_path;
228 |             }
229 |         }
230 |     }
231 | 
232 |     // Append device file, library file, file list together
233 |     list<string> files_arr;
234 |     files_arr.insert(files_arr.end(),devs.begin(),devs.end());
235 |     files_arr.insert(files_arr.end(),libs.begin(),libs.end());
236 | 
237 |     // Check validity of Src, Dst set
238 |     for(auto it : files){
239 |         files_arr.push_back(it.at(0));
240 |         if(it.at(1).empty()){
241 |             cerr << "ERROR [" << device.getName() << "] Destination for [" << it.at(0) << "] is not specified\n";
242 |             res = false;
243 |         }
244 |     }
245 | 
246 |     // Check if the files exist
247 |     for(auto const & it : files_arr){
248 |         if( ! isFileExisting(it.c_str()) ){
249 |             cerr << "ERROR [" << device.getName() << "] File [" << it<< "] not exists\n";
250 |             res = false;
251 |         }
252 |     }
253 | 
254 |     //PCI Slot check - TODO
255 |     string slot = device.getPciSlot();
256 |     uint16_t vendorID = device.getVendorID();
257 |     uint16_t deviceID = device.getDeviceID();
258 |     uint16_t subVendorID = device.getSubVendorID();
259 |     uint16_t subDeviceID = device.getSubDeviceID();
260 | 
261 |     return res;
262 | }


--------------------------------------------------------------------------------