├── python ├── src │ ├── __init__.py │ └── mlsamples │ │ ├── __init__.py │ │ ├── misc │ │ ├── __init__.py │ │ └── utils.py │ │ ├── task │ │ ├── __init__.py │ │ ├── pose │ │ │ ├── __init__.py │ │ │ ├── yolo.py │ │ │ ├── pose_interface.py │ │ │ └── detectron.py │ │ ├── detection │ │ │ ├── __init__.py │ │ │ ├── yolo.py │ │ │ ├── detect_interface.py │ │ │ └── detectron.py │ │ └── segmentation │ │ │ ├── __init__.py │ │ │ ├── yolo.py │ │ │ ├── detectron.py │ │ │ ├── segment_interface.py │ │ │ └── sgdepth.py │ │ ├── engine │ │ ├── __init__.py │ │ ├── engine_interface.py │ │ ├── detection.py │ │ ├── pose.py │ │ └── segmentation.py │ │ ├── pipeline │ │ ├── __init__.py │ │ └── builder.py │ │ ├── visual │ │ ├── __init__.py │ │ ├── visual_interface.py │ │ ├── segmentation.py │ │ ├── pose.py │ │ └── detection.py │ │ └── run.py ├── pyproject.toml ├── README.md ├── requirements.txt └── LICENSE ├── cpp ├── src │ ├── minimal.cpp │ ├── mlsamples │ │ ├── loader.cpp │ │ ├── visual │ │ │ ├── drawer.h │ │ │ └── drawer.cpp │ │ ├── pipeline.cpp │ │ └── task │ │ │ ├── yoloutils.h │ │ │ ├── detection.cpp │ │ │ ├── segmentation.cpp │ │ │ └── yoloutils.cpp │ └── run.cpp ├── include │ └── mlsamples │ │ ├── loader.h │ │ ├── task │ │ ├── pose │ │ │ ├── yolo.h │ │ │ └── interface.h │ │ ├── detection │ │ │ ├── yolo.h │ │ │ └── interface.h │ │ └── segmentation │ │ │ ├── segyolo.h │ │ │ └── seginterface.h │ │ └── pipeline.h ├── CMakeLists.txt └── README.md ├── README.md ├── .gitignore └── LICENSE /python/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/misc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/engine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/pose/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/visual/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/detection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/segmentation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cpp/src/minimal.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int main() { 5 | torch::Tensor tensor = torch::rand({2, 3}); 6 | std::cout << tensor << std::endl; 7 | } 8 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/loader.h: -------------------------------------------------------------------------------- 1 | //! \brief mlsamples torchscript model loader 2 | 3 | #ifndef LOADER_H 4 | #define LOADER_H 5 | 6 | #include 7 | #include 8 | 9 | namespace mlsamples { 10 | torch::jit::Module load(std::filesystem::path model_p); 11 | } 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /python/src/mlsamples/visual/visual_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief visual interface 3 | """ 4 | 5 | from abc import ABC, abstractmethod 6 | import torch 7 | 8 | 9 | class BaseVisual(ABC): 10 | """""" 11 | 12 | @abstractmethod 13 | def draw(self, engine_result) -> torch.Tensor: 14 | """""" 15 | raise NotImplementedError 16 | -------------------------------------------------------------------------------- /python/src/mlsamples/engine/engine_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief base engine interface 3 | """ 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | from typing import Iterator 7 | 8 | 9 | class BaseEngine(ABC): 10 | """""" 11 | 12 | @abstractmethod 13 | def run(self, video: Path) -> Iterator: 14 | raise NotImplementedError 15 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/pose/yolo.h: -------------------------------------------------------------------------------- 1 | #ifndef YOLO_H 2 | #define YOLO_H 3 | #include "interface.h" 4 | 5 | // 6 | #include 7 | #include 8 | namespace mlsamples { 9 | 10 | class Yolo : public PoseEstimator { 11 | public: 12 | Yolo(); 13 | ~Yolo(); 14 | virtual std::vector 15 | run(std::filesystem::path video) override; 16 | 17 | private: 18 | struct Impl; 19 | std::unique_ptr impl; 20 | }; 21 | } // namespace mlsamples 22 | #endif 23 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/loader.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include // One-stop header. 3 | 4 | namespace mlsamples { 5 | 6 | torch::jit::Module load(std::filesystem::path model_p) { 7 | 8 | torch::jit::script::Module m; 9 | try { 10 | std::string s = model_p.string(); 11 | m = torch::jit::load(s); 12 | } catch (const c10::Error &e) { 13 | std::cerr << "error loading the model:" << std::endl; 14 | throw std::runtime_error(e.what()); 15 | } 16 | return m; 17 | } 18 | } // namespace mlsamples 19 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/detection/yolo.h: -------------------------------------------------------------------------------- 1 | #ifndef YOLO_H 2 | #define YOLO_H 3 | #include "interface.h" 4 | 5 | // 6 | #include 7 | #include 8 | namespace mlsamples { 9 | 10 | namespace detection { 11 | 12 | class Yolo : public Detector { 13 | public: 14 | Yolo(); 15 | ~Yolo(); 16 | virtual std::vector 17 | run(std::filesystem::path video) override; 18 | 19 | private: 20 | struct Impl; 21 | std::unique_ptr impl; 22 | }; 23 | } // namespace detection 24 | } // namespace mlsamples 25 | #endif 26 | -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "mlsamples" 7 | version = "0.0.1" 8 | authors = [ 9 | {name = "D-K-E", email = "22693287+D-K-E@users.noreply.github.com"}, 10 | ] 11 | description = "Sample Deep Learning Project" 12 | readme = "README.md" 13 | requires-python = ">=3.7" 14 | keywords = ["deep-learning", "transfer-learning", "computer-vision"] 15 | license = {file = "LICENSE"} 16 | classifiers = [ 17 | "Programming Language :: Python :: 3", 18 | ] 19 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/visual/drawer.h: -------------------------------------------------------------------------------- 1 | #ifndef DRAWER_H 2 | #define DRAWER_H 3 | // basic drawing functions 4 | #include 5 | // #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace mlsamples { 11 | 12 | std::vector 13 | draw(const std::vector &ds); 14 | 15 | std::vector 16 | draw(const std::vector &masks); 17 | 18 | } // namespace mlsamples 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/segmentation/segyolo.h: -------------------------------------------------------------------------------- 1 | #ifndef SEGYOLO_H 2 | #define SEGYOLO_H 3 | 4 | #include "seginterface.h" 5 | 6 | // 7 | #include 8 | #include 9 | 10 | namespace mlsamples { 11 | namespace segmentation { 12 | 13 | class Yolo : public Segmenter { 14 | public: 15 | Yolo(); 16 | ~Yolo(); 17 | virtual std::vector 18 | run(std::filesystem::path video) override; 19 | 20 | private: 21 | struct Impl; 22 | std::unique_ptr impl; 23 | }; 24 | 25 | } // namespace segmentation 26 | } // namespace mlsamples 27 | #endif 28 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/pipeline.h: -------------------------------------------------------------------------------- 1 | #ifndef PIPELINE_H 2 | #define PIPELINE_H 3 | 4 | #include 5 | #include 6 | namespace mlsamples { 7 | 8 | enum class Task { 9 | SEGMENTATION = 1, 10 | DETECTION = 2, 11 | POSE_ESTIMATION = 3, 12 | }; 13 | enum class Backend { YOLO = 1 }; 14 | 15 | class Pipeline { 16 | public: 17 | Pipeline(Task t, Backend b, std::filesystem::path out_p); 18 | ~Pipeline(); 19 | void run(std::filesystem::path in_video); 20 | 21 | private: 22 | struct Impl; 23 | std::unique_ptr impl; 24 | }; 25 | 26 | } // namespace mlsamples 27 | #endif 28 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/pose/interface.h: -------------------------------------------------------------------------------- 1 | #ifndef INTERFACE_H 2 | #define INTERFACE_H 3 | 4 | #include 5 | // 6 | 7 | #include 8 | #include 9 | #include 10 | namespace mlsamples { 11 | struct Keypoints { 12 | Keypoints() = delete; 13 | Keypoints(torch::Tensor f, 14 | std::vector> ps); 15 | torch::Tensor frame; 16 | std::vector> keypoints; 17 | }; 18 | 19 | class PoseEstimator { 20 | public: 21 | virtual std::vector 22 | run(std::filesystem::path video) = 0; 23 | }; 24 | } // namespace mlsamples 25 | #endif 26 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/detection/interface.h: -------------------------------------------------------------------------------- 1 | #ifndef INTERFACE_H 2 | #define INTERFACE_H 3 | 4 | #include 5 | #include 6 | // 7 | 8 | #include 9 | #include 10 | namespace mlsamples { 11 | namespace detection { 12 | struct Detection { 13 | Detection() = delete; 14 | Detection(cv::Mat f, const std::vector& b); 15 | cv::Mat frame; 16 | std::vector bboxes; 17 | }; 18 | 19 | class Detector { 20 | public: 21 | virtual std::vector 22 | run(std::filesystem::path video) = 0; 23 | }; 24 | } // namespace detection 25 | } // namespace mlsamples 26 | #endif 27 | -------------------------------------------------------------------------------- /cpp/include/mlsamples/task/segmentation/seginterface.h: -------------------------------------------------------------------------------- 1 | #ifndef SEGINTERFACE_H 2 | #define SEGINTERFACE_H 3 | 4 | #include 5 | // 6 | 7 | #include 8 | #include 9 | #include 10 | namespace mlsamples { 11 | namespace segmentation { 12 | 13 | struct Mask { 14 | Mask() = delete; 15 | Mask(cv::Mat f, 16 | const std::vector>> 17 | &m); 18 | 19 | cv::Mat frame; 20 | std::vector>> masks; 21 | }; 22 | 23 | class Segmenter { 24 | public: 25 | virtual std::vector 26 | run(std::filesystem::path video) = 0; 27 | }; 28 | } // namespace segmentation 29 | } // namespace mlsamples 30 | #endif 31 | -------------------------------------------------------------------------------- /python/src/mlsamples/engine/detection.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief a detection engine based on given backend 3 | """ 4 | 5 | from mlsamples.engine.engine_interface import BaseEngine 6 | from mlsamples.misc.utils import Backend 7 | from mlsamples.misc.utils import is_type 8 | from mlsamples.task.detection.yolo import YoloDetector 9 | from mlsamples.task.detection.detectron import DetectronDetector 10 | from mlsamples.task.detection.detect_interface import Detection 11 | from typing import Iterator 12 | from pathlib import Path 13 | 14 | 15 | class DetectionEngine(BaseEngine): 16 | """""" 17 | 18 | available_backends = { 19 | Backend.YOLO: YoloDetector, 20 | Backend.DETECTRON: DetectronDetector, 21 | } 22 | 23 | def __init__(self, backend: Backend): 24 | """""" 25 | is_type(backend, "backend", Backend, True) 26 | if backend not in DetectionEngine.available_backends: 27 | raise ValueError(f"str(backend) is not available") 28 | self.model = DetectionEngine.available_backends[backend]() 29 | 30 | def run(self, video: Path) -> Iterator[Detection]: 31 | """""" 32 | return self.model.detect(video) 33 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/detection/yolo.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief yolo based instance detector 3 | """ 4 | from mlsamples.task.detection.detect_interface import Detection, Detector 5 | from mlsamples.misc.utils import is_optional_type, load_yolo, Task 6 | from pathlib import Path 7 | from collections.abc import Iterator 8 | import numpy as np 9 | import torch 10 | 11 | 12 | class YoloDetection(Detection): 13 | """""" 14 | 15 | def __init__(self, result): 16 | frame = result.orig_img 17 | if isinstance(frame, np.ndarray): 18 | frame = torch.tensor(frame) 19 | boxes = result.boxes.xywh 20 | if isinstance(boxes, np.ndarray): 21 | boxes = torch.tensor(boxes) 22 | super().__init__(frame=frame, boxes=boxes) 23 | 24 | 25 | class YoloDetector(Detector): 26 | """""" 27 | 28 | def __init__(self): 29 | """""" 30 | self.model = load_yolo(Task.DETECTION) 31 | 32 | def detect(self, video: Path) -> Iterator[Detection]: 33 | """""" 34 | results = self.model(str(video), device="cpu", stream=True) 35 | for result in results: 36 | d = YoloDetection(result) 37 | yield d 38 | -------------------------------------------------------------------------------- /python/src/mlsamples/engine/pose.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief a pose estimation engine based on given backend 3 | """ 4 | 5 | from mlsamples.engine.engine_interface import BaseEngine 6 | from mlsamples.misc.utils import Backend 7 | from mlsamples.misc.utils import is_type 8 | from mlsamples.task.pose.yolo import YoloPoseEstimator 9 | from mlsamples.task.pose.detectron import DetectronPoseEstimator 10 | from mlsamples.task.pose.pose_interface import Keypoints 11 | from typing import Iterator 12 | from pathlib import Path 13 | 14 | 15 | class PoseEstimationEngine(BaseEngine): 16 | """""" 17 | 18 | available_backends = { 19 | Backend.YOLO: YoloPoseEstimator, 20 | Backend.DETECTRON: DetectronPoseEstimator, 21 | } 22 | 23 | def __init__(self, backend: Backend): 24 | """""" 25 | is_type(backend, "backend", Backend, True) 26 | if backend not in PoseEstimationEngine.available_backends: 27 | raise ValueError(f"str(backend) is not available") 28 | self.model = PoseEstimationEngine.available_backends[backend]() 29 | 30 | def run(self, video: Path) -> Iterator[Keypoints]: 31 | """""" 32 | return self.model.estimate_poses(video) 33 | -------------------------------------------------------------------------------- /python/src/mlsamples/engine/segmentation.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief a segmentation engine based on given backend 3 | """ 4 | 5 | from mlsamples.engine.engine_interface import BaseEngine 6 | from mlsamples.misc.utils import Backend 7 | from mlsamples.misc.utils import is_type 8 | from mlsamples.task.segmentation.yolo import YoloSegmenter 9 | from mlsamples.task.segmentation.detectron import DetectronSegmenter 10 | from mlsamples.task.segmentation.segment_interface import SegmentationMask 11 | from typing import Iterator 12 | from pathlib import Path 13 | 14 | 15 | class SegmentationEngine(BaseEngine): 16 | """""" 17 | 18 | available_backends = { 19 | Backend.YOLO: YoloSegmenter, 20 | Backend.DETECTRON: DetectronSegmenter, 21 | } 22 | 23 | def __init__(self, backend: Backend): 24 | """""" 25 | is_type(backend, "backend", Backend, True) 26 | if backend not in SegmentationEngine.available_backends: 27 | raise ValueError(f"str(backend) is not available") 28 | self.model = SegmentationEngine.available_backends[backend]() 29 | 30 | def run(self, video: Path) -> Iterator[SegmentationMask]: 31 | """""" 32 | return self.model.segment(video) 33 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/segmentation/yolo.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief yolo based instance segmenter 3 | """ 4 | from mlsamples.task.segmentation.segment_interface import SegmentationMask, Segmenter 5 | from pathlib import Path 6 | from collections.abc import Iterator 7 | 8 | from mlsamples.misc.utils import Task 9 | from mlsamples.misc.utils import load_yolo 10 | import numpy as np 11 | import torch 12 | 13 | 14 | class YoloMask(SegmentationMask): 15 | """""" 16 | 17 | def __init__(self, result): 18 | frame = result.orig_img 19 | if isinstance(frame, np.ndarray): 20 | frame = torch.tensor(frame) 21 | points = result.masks.xy 22 | masks = [] 23 | for point_arr in points: 24 | ps = [] 25 | for point in point_arr: 26 | x = int(point[0]) 27 | y = int(point[1]) 28 | ps.append((x, y)) 29 | masks.append(ps) 30 | super().__init__(masks=masks, frame=frame) 31 | 32 | 33 | class YoloSegmenter(Segmenter): 34 | """""" 35 | 36 | def __init__(self): 37 | self.model = load_yolo(Task.SEGMENTATION) 38 | 39 | def segment(self, video: Path) -> Iterator[SegmentationMask]: 40 | """""" 41 | results = self.model(str(video), device="cpu", stream=True) 42 | for result in results: 43 | mask = YoloMask(result) 44 | yield mask 45 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/pose/yolo.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief yolo based pose key points 3 | """ 4 | from mlsamples.task.pose.pose_interface import Keypoints 5 | from mlsamples.task.pose.pose_interface import PoseEstimator 6 | from mlsamples.misc.utils import is_optional_type 7 | from mlsamples.misc.utils import Task 8 | from mlsamples.misc.utils import load_yolo 9 | from pathlib import Path 10 | from collections.abc import Iterator 11 | import numpy as np 12 | import torch 13 | 14 | 15 | class YoloKeypoints(Keypoints): 16 | """""" 17 | 18 | def __init__(self, result): 19 | frame = result.orig_img 20 | if isinstance(frame, np.ndarray): 21 | frame = torch.tensor(frame) 22 | 23 | keypoints = [] 24 | for points in result.keypoints.xy: 25 | for point in points: 26 | x = point[0].item() 27 | y = point[1].item() 28 | keypoints.append((x, y)) 29 | 30 | super().__init__(keypoints=keypoints, frame=frame) 31 | 32 | 33 | class YoloPoseEstimator(PoseEstimator): 34 | """""" 35 | 36 | def __init__(self): 37 | self.model = load_yolo(Task.POSE_ESTIMATION) 38 | 39 | def estimate_poses(self, video: Path) -> Iterator[Keypoints]: 40 | """""" 41 | results = self.model(str(video), device="cpu", stream=True) 42 | for result in results: 43 | mask = YoloKeypoints(result) 44 | yield mask 45 | -------------------------------------------------------------------------------- /cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.18 FATAL_ERROR) 2 | project(aidev) 3 | 4 | find_package(Torch REQUIRED PATHS "lib/libtorch") 5 | 6 | find_package(OpenCV REQUIRED) 7 | 8 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") 9 | set(CMAKE_BUILD_TYPE "RelWithDebInfo") 10 | 11 | add_executable(min.out 12 | "src/minimal.cpp" 13 | ) 14 | 15 | target_link_libraries(min.out "${TORCH_LIBRARIES}") 16 | set_property(TARGET min.out PROPERTY CXX_STANDARD 17) 17 | 18 | include(FetchContent) 19 | FetchContent_Declare( 20 | argparse 21 | GIT_REPOSITORY https://github.com/p-ranav/argparse.git 22 | ) 23 | FetchContent_MakeAvailable(argparse) 24 | 25 | 26 | add_executable(run.out 27 | "src/run.cpp" 28 | "src/mlsamples/loader.cpp" 29 | "src/mlsamples/pipeline.cpp" 30 | "src/mlsamples/visual/drawer.cpp" 31 | "src/mlsamples/task/detection.cpp" 32 | "src/mlsamples/task/yoloutils.cpp" 33 | "src/mlsamples/task/segmentation.cpp" 34 | ) 35 | 36 | target_link_libraries(run.out "${TORCH_LIBRARIES}") 37 | target_link_libraries(run.out argparse) 38 | target_link_libraries(run.out ${OpenCV_LIBRARIES}) 39 | 40 | target_include_directories(run.out PUBLIC "include") 41 | 42 | target_include_directories(run.out PRIVATE "${OpenCV_INCLUDE_DIRS}") 43 | 44 | set_property(TARGET run.out PROPERTY CXX_STANDARD 17) 45 | target_compile_definitions( 46 | run.out 47 | PRIVATE 48 | ASSET_DIR="${CMAKE_CURRENT_SOURCE_DIR}/assets" 49 | ) 50 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/detection/detect_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief detection interface for all backends 3 | """ 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | from mlsamples.misc.utils import is_optional_type 7 | from mlsamples.misc.utils import is_type 8 | from mlsamples.misc.utils import FrameContainer 9 | 10 | from collections.abc import Iterator 11 | import torch 12 | import numpy as np 13 | 14 | 15 | class BaseDetection(ABC): 16 | """""" 17 | 18 | @property 19 | @abstractmethod 20 | def bboxes(self) -> torch.Tensor: 21 | """""" 22 | raise NotImplementedError 23 | 24 | @property 25 | @abstractmethod 26 | def frame(self) -> np.ndarray: 27 | """""" 28 | raise NotImplementedError 29 | 30 | 31 | class Detection(FrameContainer, BaseDetection): 32 | def __init__(self, boxes: torch.Tensor, frame: np.ndarray): 33 | super().__init__(frame=frame) 34 | is_optional_type(boxes, "boxes", torch.Tensor, True) 35 | self._bboxes = boxes 36 | 37 | def set_bboxes(self, bboxes: torch.Tensor): 38 | """""" 39 | is_type(bboxes, "bboxes", torch.Tensor, True) 40 | self._bboxes = bboxes 41 | 42 | @property 43 | def bboxes(self): 44 | """""" 45 | if self._bboxes is None: 46 | raise ValueError("bboxes is none") 47 | return self._bboxes 48 | 49 | 50 | class Detector(ABC): 51 | """""" 52 | 53 | @abstractmethod 54 | def detect(self, video: Path) -> Iterator[Detection]: 55 | """""" 56 | raise NotImplementedError 57 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/segmentation/detectron.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief detectron based instance segmenter 3 | """ 4 | from mlsamples.task.segmentation.segment_interface import SegmentationMask, Segmenter 5 | from mlsamples.misc.utils import load_detectron 6 | from mlsamples.misc.utils import Task 7 | from pathlib import Path 8 | from collections.abc import Iterator 9 | from torchvision.io import read_video 10 | import numpy as np 11 | import torch 12 | 13 | 14 | class DetectronMask(SegmentationMask): 15 | """""" 16 | 17 | def __init__(self, result: dict): 18 | frame = result["frame"] 19 | if isinstance(frame, np.ndarray): 20 | frame = torch.tensor(frame) 21 | pmask = result["pred_masks"] 22 | pmask_nz = torch.nonzero(pmask) 23 | masks = [[] for _ in range(pmask.shape[0])] 24 | for p in pmask_nz: 25 | n, y, x = p 26 | masks[n].append((x, y)) 27 | super().__init__(masks=masks, frame=frame) 28 | 29 | 30 | class DetectronSegmenter(Segmenter): 31 | """""" 32 | 33 | def __init__(self): 34 | self.model = load_detectron(Task.SEGMENTATION) 35 | 36 | def segment(self, video: Path) -> Iterator[SegmentationMask]: 37 | """""" 38 | v = read_video(str(video)) 39 | clip = v[0] 40 | T, Height, Width, Channel = clip.shape 41 | for t in range(T): 42 | frame = clip[t, :].numpy() 43 | preds = self.model(frame) 44 | result = dict(pred_masks=preds["instances"].pred_masks, frame=frame) 45 | mask = DetectronMask(result) 46 | yield mask 47 | -------------------------------------------------------------------------------- /python/src/mlsamples/visual/segmentation.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief segmentation visual 3 | """ 4 | from mlsamples.visual.visual_interface import BaseVisual 5 | from mlsamples.task.segmentation.segment_interface import SegmentationMask 6 | from typing import Iterator 7 | import torch 8 | import cv2 9 | import random 10 | import numpy as np 11 | 12 | 13 | class SegmentationVisual(BaseVisual): 14 | """""" 15 | 16 | random_colors = [ 17 | # [0, 0, 0], # 18 | # [255, 255, 255], # 19 | # [255, 0, 0], # 20 | # [0, 255, 0], # 21 | # [0, 0, 255], # 22 | # [255, 255, 0], # 23 | # [0, 255, 255], # 24 | [255, 0, 255], # 25 | ] 26 | 27 | def draw(self, engine_result: Iterator[SegmentationMask]) -> torch.Tensor: 28 | """""" 29 | result = [] 30 | alpha = 0.7 31 | beta = 1.0 - alpha 32 | for d in engine_result: 33 | frame = d.frame 34 | masks = d.masks 35 | arr = frame.numpy() 36 | frame_img = arr.copy() # cv2.cvtColor(arr, cv2.COLOR_RGB2BGR) 37 | temp = np.zeros_like(frame_img) 38 | 39 | for mask in masks: 40 | color = random.choice(SegmentationVisual.random_colors) 41 | m = np.array(mask) 42 | temp = cv2.fillConvexPoly(temp, m, color) 43 | # 44 | frame_img = cv2.addWeighted(frame_img, alpha, temp, beta, 0.0) 45 | # 46 | # frame_img = cv2.cvtColor(frame_img, cv2.COLOR_BGR2RGB) 47 | result.append(frame_img) 48 | # 49 | return torch.tensor(result) 50 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # ML Samples 2 | 3 | Sample code for various computer vision related deep learning tasks. 4 | 5 | ## Build Instructions for Ubuntu 22.04 6 | 7 | - Create a virtual environment: `python -m venv venv` 8 | - Activate the virtual environment: `source venv/bin/activate` 9 | - Install dependencies: `pip install -r requirements.txt` 10 | - Install the package: `pip install .` 11 | 12 | You should be able to launch the `run.py` under `src` directory now. 13 | 14 | ## Usage example 15 | 16 | To access the options from the command line interface, launch the `run.py` 17 | from project directory (directory containing `pyproject.toml` file). For 18 | example: 19 | 20 | - Activate virtual environment: `source venv/bin/activate` 21 | - Run the program to see options: `python src/mlsamples/run.py -h` 22 | ``` 23 | usage: Simple ai assisted video analysis toolkit [-h] --backend {yolo,detectron} --task {segment,detect,pose} --video VIDEO 24 | [--save_name SAVE_NAME] 25 | 26 | options: 27 | -h, --help show this help message and exit 28 | --backend {yolo,detectron} 29 | --task {segment,detect,pose} 30 | --video VIDEO path to input video e.g. face.mp4 31 | --save_name SAVE_NAME 32 | output name for analyzed video e.g. output.mp4 33 | ``` 34 | 35 | A choose a backend and a task and pass the path of a video directory: 36 | ``` 37 | $ python src/mlsamples/run.py --backend detectron --task segment --video ../data/out02.mp4 --save_name detectron_segment.mp4 38 | ``` 39 | This would produce a `detectron_segment.mp4` file inside the project 40 | directory. 41 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/detection/detectron.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief detectron based instance segmenter 3 | """ 4 | from pathlib import Path 5 | from mlsamples.task.detection.detect_interface import Detection, Detector 6 | from mlsamples.misc.utils import load_detectron 7 | from mlsamples.misc.utils import Task 8 | from collections.abc import Iterator 9 | from torchvision.io import read_video 10 | import torch 11 | import numpy as np 12 | 13 | 14 | class DetectronDetection(Detection): 15 | """""" 16 | 17 | def __init__(self, result: dict): 18 | frame = result["frame"] 19 | if isinstance(frame, np.ndarray): 20 | frame = torch.tensor(frame) 21 | 22 | boxes = [] 23 | pboxes = result["pred_boxes"].tensor 24 | boxes = torch.zeros_like(pboxes) 25 | boxes[:, 0] = pboxes[:, 0] 26 | boxes[:, 1] = pboxes[:, 1] 27 | boxes[:, 2] = pboxes[:, 2] - pboxes[:, 0] 28 | boxes[:, 3] = pboxes[:, 3] - pboxes[:, 1] 29 | 30 | super().__init__(frame=frame, boxes=boxes) 31 | 32 | 33 | class DetectronDetector(Detector): 34 | """""" 35 | 36 | def __init__(self): 37 | self.model = load_detectron(Task.DETECTION) 38 | 39 | def detect(self, video: Path) -> Iterator[Detection]: 40 | """""" 41 | v = read_video(str(video)) 42 | clip = v[0] 43 | T, Height, Width, Channel = clip.shape 44 | for t in range(T): 45 | frame = clip[t, :].numpy() 46 | preds = self.model(frame) 47 | result = dict(pred_boxes=preds["instances"].pred_boxes, frame=frame) 48 | d = DetectronDetection(result) 49 | yield d 50 | -------------------------------------------------------------------------------- /python/src/mlsamples/visual/pose.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief keypoints visual 3 | """ 4 | from mlsamples.visual.visual_interface import BaseVisual 5 | from mlsamples.task.pose.pose_interface import Keypoints 6 | from typing import Iterator 7 | import torch 8 | import cv2 9 | import numpy as np 10 | import random 11 | 12 | 13 | class KeypointsVisual(BaseVisual): 14 | """""" 15 | 16 | random_colors = [ 17 | # [0, 0, 0], # 18 | # [255, 255, 255], # 19 | # [255, 0, 0], # 20 | # [0, 255, 0], # 21 | # [0, 0, 255], # 22 | # [255, 255, 0], # 23 | # [0, 255, 255], # 24 | [255, 0, 255], # 25 | ] 26 | 27 | def draw(self, engine_result: Iterator[Keypoints]) -> torch.Tensor: 28 | """""" 29 | result = [] 30 | for d in engine_result: 31 | frame = d.frame 32 | keypoints = d.keypoints 33 | arr = frame.numpy() 34 | frame_img = arr.copy() # cv2.cvtColor(arr, cv2.COLOR_RGB2BGR) 35 | temp = np.zeros_like(frame_img) 36 | color = random.choice(KeypointsVisual.random_colors) 37 | for point in keypoints: 38 | x, y = list(map(int, point)) 39 | frame_img = cv2.circle( 40 | frame_img, # img 41 | (x, y), # center 42 | 2, # radius 43 | color, # white 44 | -1, # thickness 45 | ) 46 | # 47 | # frame_img = cv2.cvtColor(frame_img, cv2.COLOR_BGR2RGB) 48 | result.append(frame_img) 49 | # 50 | return torch.tensor(result) 51 | -------------------------------------------------------------------------------- /cpp/README.md: -------------------------------------------------------------------------------- 1 | # Building on Linux 2 | 3 | ## Dependencies 4 | 5 | - OpenCV 4.5+ 6 | - libtorch 7 | - C++17 compiler 8 | - CMake 3.18+ 9 | 10 | Rest of the dependencies are downloaded with FetchContent. 11 | 12 | ## Building 13 | 14 | - `mkdir build` 15 | - `cd build` 16 | - `cmake ..` 17 | - `make` 18 | 19 | These commands should create two executables: `min.out` and `run.out`. 20 | 21 | Run `min.out` to test whether `torch` is linked correctly or not. 22 | 23 | 24 | ## Tested Toolchain 25 | 26 | ``` 27 | -- The C compiler identification is GNU 11.4.0 28 | -- The CXX compiler identification is GNU 11.4.0 29 | 30 | -- Found Torch: ./mlsamples/cpp/lib/libtorch/lib/libtorch.so 31 | -- Found OpenCV: (found version "4.5.4") 32 | ``` 33 | 34 | # Usage 35 | 36 | Once you built the program, you launch the `run.out` to see available options: 37 | 38 | - `./run.out -h`: This should give you something like the following: 39 | 40 | ``` 41 | Usage: ai task runner [--help] [--version] [--backend VAR] --task VAR --video VAR --save_name VAR 42 | 43 | Optional arguments: 44 | -h, --help shows help message and exits 45 | -v, --version prints version information and exits 46 | --backend [nargs=0..1] [default: "yolo"] 47 | --task [nargs=0..1] [default: "segment"] 48 | --video path to input video [required] 49 | --save_name save name for output video, e.g. out.mp4 [nargs=0..1] [default: "out.mp4"] 50 | ``` 51 | There is one backend (`yolo`) and two tasks (`segment`, `detect`) that 52 | are available. To launch `segment` task with `yolo` backend on a video: 53 | 54 | - `$ ./run.out --task segment --video ../../data/out02.mp4 --save_name segment03.mp4` 55 | -------------------------------------------------------------------------------- /python/src/mlsamples/visual/detection.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief detection visual 3 | """ 4 | from mlsamples.visual.visual_interface import BaseVisual 5 | from mlsamples.task.detection.detect_interface import Detection 6 | from typing import Iterator 7 | import torch 8 | import cv2 9 | import random 10 | 11 | 12 | class DetectionVisual(BaseVisual): 13 | """""" 14 | 15 | random_colors = [ 16 | # [0, 0, 0], # 17 | # [255, 255, 255], # 18 | # [255, 0, 0], # 19 | # [0, 255, 0], # 20 | # [0, 0, 255], # 21 | # [255, 255, 0], # 22 | # [0, 255, 255], # 23 | [255, 0, 255], # 24 | ] 25 | 26 | def draw(self, engine_result: Iterator[Detection]) -> torch.Tensor: 27 | """""" 28 | result = [] 29 | for d in engine_result: 30 | frame = d.frame 31 | bboxes = d.bboxes.numpy() 32 | arr = frame.numpy() 33 | frame_img = arr.copy() # cv2.cvtColor(arr, cv2.COLOR_RGB2BGR) 34 | rects = [] 35 | for i in range(bboxes.shape[0]): 36 | bbox = list(map(int, bboxes[i, :].flatten().tolist())) 37 | x = bbox[0] 38 | y = bbox[1] 39 | w = bbox[2] 40 | h = bbox[3] 41 | rect = (x, y, w, h) 42 | color = random.choice(DetectionVisual.random_colors) 43 | frame_img = cv2.rectangle(frame_img, rect, color, 2) 44 | # rects.append(frame_img) 45 | # 46 | # frame_img = cv2.cvtColor(frame_img, cv2.COLOR_BGR2RGB) 47 | result.append(frame_img) 48 | # 49 | return torch.tensor(result) 50 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/segmentation/segment_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief segmentation interface for all backends 3 | """ 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | 7 | from typing import List, Optional, Tuple 8 | import torch 9 | from collections.abc import Iterator 10 | from mlsamples.misc.utils import FrameContainer 11 | from mlsamples.misc.utils import is_optional_type 12 | 13 | 14 | class BaseMask(ABC): 15 | """""" 16 | 17 | @property 18 | @abstractmethod 19 | def masks(self) -> List[List[Tuple[int, int]]]: 20 | """""" 21 | raise NotImplementedError 22 | 23 | @property 24 | @abstractmethod 25 | def frame(self) -> torch.Tensor: 26 | """""" 27 | raise NotImplementedError 28 | 29 | 30 | class SegmentationMask(FrameContainer, BaseMask): 31 | def __init__( 32 | self, 33 | masks: Optional[List[List[Tuple[int, int]]]] = None, 34 | frame: Optional[torch.Tensor] = None, 35 | ): 36 | super().__init__(frame=frame) 37 | is_optional_type(masks, "masks", list, True) 38 | self._masks = masks 39 | 40 | def set_masks(self, masks: List[List[Tuple[int, int]]]): 41 | """""" 42 | is_type(masks, "masks", list, True) 43 | all(is_type(m, "m", list, True) for m in masks) 44 | self._masks = masks 45 | 46 | @property 47 | def masks(self): 48 | """""" 49 | if self._masks is None: 50 | raise ValueError("masks is none") 51 | return self._masks 52 | 53 | 54 | class Segmenter(ABC): 55 | """""" 56 | 57 | @abstractmethod 58 | def segment(self, video: Path) -> Iterator[SegmentationMask]: 59 | """""" 60 | raise NotImplementedError 61 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/pose/pose_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief pose estimation interface 3 | """ 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | 7 | from typing import List, Optional, Tuple 8 | import torch 9 | from collections.abc import Iterator 10 | from mlsamples.misc.utils import is_optional_type 11 | from mlsamples.misc.utils import is_type 12 | from mlsamples.misc.utils import FrameContainer 13 | 14 | 15 | class BaseKeypoints(ABC): 16 | """""" 17 | 18 | @property 19 | @abstractmethod 20 | def keypoints(self) -> List[Tuple[int, int]]: 21 | """""" 22 | raise NotImplementedError 23 | 24 | @property 25 | @abstractmethod 26 | def frame(self) -> torch.Tensor: 27 | """""" 28 | raise NotImplementedError 29 | 30 | 31 | class Keypoints(FrameContainer, BaseKeypoints): 32 | def __init__( 33 | self, 34 | keypoints: Optional[torch.Tensor] = None, 35 | frame: Optional[torch.Tensor] = None, 36 | ): 37 | super().__init__(frame=frame) 38 | self._keypoints = keypoints 39 | 40 | def set_keypoints(self, keypoints: List[Tuple[int, int]]): 41 | """""" 42 | is_type(keypoints, "keypoints", list, True) 43 | all(is_type(k, "keypoints", tuple, True) for k in keypoints) 44 | self._keypoints = keypoints 45 | 46 | @property 47 | def keypoints(self) -> List[Tuple[int, int]]: 48 | if self._keypoints is None: 49 | raise ValueError("keypoints is none") 50 | return self._keypoints 51 | 52 | 53 | class PoseEstimator(ABC): 54 | """""" 55 | 56 | @abstractmethod 57 | def estimate_poses(self, video: Path) -> Iterator[Keypoints]: 58 | """""" 59 | raise NotImplementedError 60 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/visual/drawer.cpp: -------------------------------------------------------------------------------- 1 | #include "drawer.h" 2 | 3 | // other stuff 4 | #include 5 | #include 6 | #include 7 | using namespace torch::indexing; 8 | 9 | namespace mlsamples { 10 | // 11 | std::vector 12 | draw(const std::vector &ds) { 13 | // 14 | std::vector results; 15 | for (auto d : ds) { 16 | cv::Mat frame = d.frame; 17 | std::vector boxes = d.bboxes; 18 | for (int i = 0; i < boxes.size(); ++i) { 19 | auto box = boxes[i]; 20 | cv::rectangle(frame, box, cv::Scalar(255,0,255)); 21 | } 22 | results.push_back(frame); 23 | } 24 | return results; 25 | } 26 | 27 | std::vector 28 | draw(const std::vector &masks) { 29 | 30 | std::vector results; 31 | auto to_p = [](const std::pair &point) { 32 | int x = point.first; 33 | int y = point.second; 34 | auto p = cv::Point(x, y); 35 | return p; 36 | }; 37 | for (auto mask : masks) { 38 | cv::Mat frame = mask.frame; 39 | std::vector>> 40 | ms_per_frame = mask.masks; 41 | cv::Mat temp(frame.rows, frame.cols, CV_8UC3, 42 | cv::Scalar(0)); 43 | for (int i = 0; i < ms_per_frame.size(); ++i) { 44 | std::vector> points = 45 | ms_per_frame[i]; 46 | std::vector ps(points.size()); 47 | std::transform(points.begin(), points.end(), 48 | ps.begin(), to_p); 49 | cv::fillConvexPoly(temp, ps.data(), ps.size(), 50 | cv::Scalar(255, 0, 255)); 51 | } 52 | cv::Mat result; 53 | cv::addWeighted(frame, 0.6, temp, 0.4, 0, result); 54 | results.push_back(result); 55 | } 56 | return results; 57 | } 58 | } // namespace mlsamples 59 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/pose/detectron.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief detectron based pose key points 3 | """ 4 | from mlsamples.task.pose.pose_interface import Keypoints 5 | from mlsamples.task.pose.pose_interface import PoseEstimator 6 | from mlsamples.misc.utils import load_detectron, Task 7 | from mlsamples.misc.utils import is_optional_type 8 | from collections.abc import Iterator 9 | from pathlib import Path 10 | from torchvision.io import read_video 11 | import torch 12 | import numpy as np 13 | 14 | 15 | class DetectronKeypoints(Keypoints): 16 | """""" 17 | 18 | def __init__(self, result): 19 | frame = result["frame"] 20 | if isinstance(frame, np.ndarray): 21 | frame = torch.tensor(frame) 22 | keypoints = [] 23 | for points in result["pred_keypoints"]: 24 | x = points[0].item() 25 | y = points[1].item() 26 | keypoints.append((x, y)) 27 | 28 | super().__init__(keypoints=keypoints, frame=frame) 29 | 30 | 31 | class DetectronPoseEstimator(PoseEstimator): 32 | """""" 33 | 34 | def __init__(self): 35 | self.model = load_detectron(Task.POSE_ESTIMATION) 36 | 37 | def estimate_poses(self, video: Path) -> Iterator[Keypoints]: 38 | """""" 39 | v = read_video(str(video)) 40 | clip = v[0] 41 | T, Height, Width, Channel = clip.shape 42 | for t in range(T): 43 | frame = clip[t, :].numpy() 44 | preds = self.model(frame) 45 | keypoints: torch.Tensor = preds["instances"].pred_keypoints 46 | # transform keypoints to Tensor[Number of keypoints, XY] 47 | N, num_kpts, C = keypoints.shape 48 | nkpts = torch.reshape(keypoints, (N * num_kpts, C)) 49 | kpts = nkpts[:, :2] 50 | result = dict(pred_keypoints=kpts, frame=frame) 51 | mask = DetectronKeypoints(result) 52 | yield mask 53 | -------------------------------------------------------------------------------- /python/src/mlsamples/run.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief main entrance point of command line program 3 | """ 4 | import argparse 5 | from datetime import datetime, timezone 6 | from pathlib import Path 7 | from mlsamples.pipeline.builder import Pipeline, build_pipeline 8 | from mlsamples.misc.utils import Backend, Task 9 | 10 | 11 | def run_task(backend: Backend, task: Task, in_video: Path, save_loc: Path): 12 | """""" 13 | pipe = build_pipeline(backend=backend, task=task, save_location=save_loc) 14 | pipe.run(in_video) 15 | 16 | 17 | if __name__ == "__main__": 18 | parser = argparse.ArgumentParser("Simple ai assisted video analysis toolkit") 19 | backends = dict(yolo=Backend.YOLO, detectron=Backend.DETECTRON) 20 | tasks = dict( 21 | segment=Task.SEGMENTATION, 22 | detect=Task.DETECTION, 23 | pose=Task.POSE_ESTIMATION, 24 | ) 25 | 26 | parser.add_argument( 27 | "--backend", required=True, choices=list(backends.keys()), default="yolo" 28 | ) 29 | parser.add_argument( 30 | "--task", 31 | required=True, 32 | choices=list(tasks.keys()), 33 | default="detect", 34 | ) 35 | parser.add_argument( 36 | "--video", 37 | required=True, 38 | help="path to input video e.g. face.mp4", 39 | ) 40 | now = datetime.now(timezone.utc) 41 | ts = "-".join(map(str, now.timetuple())) 42 | parser.add_argument( 43 | "--save_name", 44 | help="output name for analyzed video e.g. output.mp4", 45 | default="out" + ts + ".mp4", 46 | ) 47 | args = parser.parse_args() 48 | 49 | backend = backends[args.backend] 50 | task = tasks[args.task] 51 | in_video = Path(args.video) 52 | if not in_video.exists(): 53 | raise FileNotFoundError(f"{args.video} does not exists") 54 | run_task(backend=backend, task=task, in_video=in_video, save_loc=Path(args.save_name)) 55 | print("all done") 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DL Samples 2 | 3 | These samples should help you get started in DL based projects. 4 | The main purpose in both samples is to isolate the model code from the pipeline 5 | that would use its services. 6 | They serve as a toy deployment example in a desktop environment. 7 | 8 | There are two samples in this repository: `python` and `c++`. Essentially they 9 | are both simple AI based video analysis tools. They both support multiple 10 | tasks: 11 | 12 | - python: detection, segmentation, pose estimation 13 | - c++: detection, segmentation 14 | 15 | 16 | 17 | 18 | Python Detectron2 Segmentation 19 | https://github.com/D-K-E/mlsamples/assets/22693287/9013a8dd-0c29-4d8e-8380-929cbd2aed7c 20 | 21 | 22 | Python Yolov8 Segmentation 23 | https://github.com/D-K-E/mlsamples/assets/22693287/009c200b-6f03-4123-95fe-5b1011c31153 24 | 25 | C++ Yolov8 Detection 26 | https://github.com/D-K-E/mlsamples/assets/22693287/d6c2e986-01f9-4e3f-bf84-4828136053c1 27 | 28 | 29 | 30 | 31 | Python version supports multiple backends: 32 | 33 | - YOLOv8 34 | - Detectron2 35 | 36 | C++ version supports only YOLOv8. 37 | 38 | Python version automatically downloads needed models, but C++ version requires 39 | them to be present in the `assets` directory. 40 | 41 | Though C++ and Python is using the same backend model `YOLOv8`, they produce 42 | different outputs. This is due to preprocessing and post processing functions 43 | involved prior and after the inference. The python version uses `ultralytics` 44 | library to leverage preprocessing/postprocessing. The C++ version implements 45 | it from scratch with some help from various open sourced libraries. 46 | 47 | Each repository contains its proper build instructions. We are also providing 48 | test and reference videos in case the user wants to not only inspect the 49 | source code but also launch the program and see what it provides. 50 | These videos can be found in the `data` folder. 51 | 52 | All samples are tested with Ubuntu 22.04. 53 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/pipeline.cpp: -------------------------------------------------------------------------------- 1 | // include related 2 | #include "visual/drawer.h" 3 | // 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include 14 | 15 | namespace mlsamples { 16 | // 17 | struct Pipeline::Impl { 18 | 19 | Impl(Task t, Backend b, std::filesystem::path o) 20 | : task(t), backend(b), save_loc(o) { 21 | if (t == Task::DETECTION) { 22 | if (backend == Backend::YOLO) { 23 | detector.reset(new detection::Yolo()); 24 | } 25 | } else if (t == Task::SEGMENTATION) { 26 | if (backend == Backend::YOLO) { 27 | segmenter.reset(new segmentation::Yolo()); 28 | } 29 | } 30 | } 31 | ~Impl() = default; 32 | void run(std::filesystem::path video) { 33 | std::vector frames; 34 | if (task == Task::DETECTION) { 35 | std::vector ds = 36 | detector->run(video); 37 | frames = draw(ds); 38 | } else if (task == Task::SEGMENTATION) { 39 | std::vector ds = 40 | segmenter->run(video); 41 | frames = draw(ds); 42 | } 43 | std::string filename = save_loc.string(); 44 | int codec = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); 45 | int fps = 30; 46 | cv::VideoWriter writer(filename, codec, fps, 47 | frames[0].size(), true); 48 | for (auto frame : frames) { 49 | writer.write(frame); 50 | } 51 | } 52 | // 53 | Task task; 54 | Backend backend; 55 | std::filesystem::path save_loc; 56 | 57 | std::unique_ptr detector{nullptr}; 58 | std::unique_ptr segmenter{ 59 | nullptr}; 60 | }; 61 | 62 | Pipeline::Pipeline(Task t, Backend b, 63 | std::filesystem::path o) { 64 | auto temp = std::make_unique(t, b, o); 65 | impl = std::move(temp); 66 | } 67 | Pipeline::~Pipeline() = default; 68 | 69 | void Pipeline::run(std::filesystem::path in_video) { 70 | impl->run(in_video); 71 | } 72 | 73 | } // namespace mlsamples 74 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/task/yoloutils.h: -------------------------------------------------------------------------------- 1 | // simple utilities for yolo based funcs 2 | #ifndef YOLOUTILS_H 3 | #define YOLOUTILS_H 4 | #include 5 | #include 6 | #include 7 | #include 8 | namespace mlsamples { 9 | 10 | const int kYOLO_NETWORK_WIDTH_ = 640; 11 | const int kYOLO_NETWORK_HEIGHT_ = 640; 12 | const int kYOLO_CHANNEL_ = 3; 13 | const float kYOLO_CONFIDENCE_THRESHOLD = 0.35f; 14 | const float kYOLO_NMS_THRESHOLD = 0.4f; 15 | 16 | // taken from 17 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/YoloV8.cpp 18 | std::vector 19 | yolo_preprocess_image(const cv::Mat &image); 20 | 21 | // taken from 22 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/YoloV8.cpp 23 | cv::Rect yolo_get_rect(const cv::Size &imgSz, 24 | const std::vector &bbox); 25 | 26 | namespace detection { 27 | 28 | // inspired from 29 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/YoloV8.cpp 30 | std::vector yolo_postprocess( 31 | const std::vector> &outputs, 32 | const std::vector> &shapes, 33 | const cv::Size &frame_size); 34 | } // namespace detection 35 | 36 | namespace segmentation { 37 | struct SegOutput { 38 | SegOutput(const std::vector &bs, 39 | const cv::Mat &s, const cv::Mat &p, 40 | const cv::Rect &r); 41 | std::vector bboxes; 42 | cv::Mat segm; 43 | cv::Mat mask_proposals; 44 | cv::Rect roi; 45 | }; 46 | 47 | // inspired from 48 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/YoloV8.cpp 49 | SegOutput yolo_postprocess( 50 | const std::vector> &outputs, 51 | const std::vector> &shapes, 52 | const cv::Size &frame_size); 53 | } // namespace segmentation 54 | 55 | struct YoloV8Model { 56 | YoloV8Model(Task t); 57 | 58 | std::pair>, 59 | std::vector>> 60 | infer(std::vector inputs); 61 | 62 | torch::jit::Module model; 63 | }; 64 | 65 | } // namespace mlsamples 66 | #endif 67 | -------------------------------------------------------------------------------- /python/src/mlsamples/pipeline/builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief very simple pipeline builder 3 | """ 4 | from pathlib import Path 5 | from mlsamples.engine.engine_interface import BaseEngine 6 | from mlsamples.engine.detection import DetectionEngine 7 | from mlsamples.engine.segmentation import SegmentationEngine 8 | from mlsamples.engine.pose import PoseEstimationEngine 9 | from mlsamples.visual.detection import DetectionVisual 10 | from mlsamples.visual.segmentation import SegmentationVisual 11 | from mlsamples.visual.pose import KeypointsVisual 12 | 13 | from mlsamples.visual.visual_interface import BaseVisual 14 | from mlsamples.misc.utils import is_type 15 | from mlsamples.misc.utils import Backend, Task 16 | import torch 17 | from torchvision.io import write_video 18 | import moviepy.editor as mpy 19 | 20 | 21 | class Pipeline: 22 | """""" 23 | 24 | def __init__(self, engine: BaseEngine, visualizer: BaseVisual, save_location: Path): 25 | """""" 26 | is_type(engine, "engine", BaseEngine, True) 27 | self.engine = engine 28 | 29 | # 30 | is_type(visualizer, "visualizer", BaseVisual, True) 31 | self.visualizer = visualizer 32 | 33 | is_type(save_location, "save_location", Path, True) 34 | self.out = save_location 35 | 36 | def run(self, video: Path): 37 | """\brief Run the pipeline and save the result to location""" 38 | engine_result = self.engine.run(video) 39 | result = self.visualizer.draw(engine_result) 40 | clip = mpy.VideoFileClip(str(video)) 41 | fps = clip.fps 42 | write_video(filename=str(self.out), video_array=result, fps=fps) 43 | 44 | 45 | def build_pipeline(backend: Backend, task: Task, save_location: Path): 46 | """""" 47 | if task == Task.SEGMENTATION: 48 | engine = SegmentationEngine(backend) 49 | visual = SegmentationVisual() 50 | elif task == Task.DETECTION: 51 | engine = DetectionEngine(backend) 52 | visual = DetectionVisual() 53 | elif task == Task.POSE_ESTIMATION: 54 | engine = PoseEstimationEngine(backend) 55 | visual = KeypointsVisual() 56 | else: 57 | raise ValueError(f"unsupported task {str(task)}") 58 | pipe = Pipeline(engine=engine, visualizer=visual, save_location=save_location) 59 | return pipe 60 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/task/detection.cpp: -------------------------------------------------------------------------------- 1 | // implement detection related stuff 2 | #include "yoloutils.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace mlsamples { 14 | namespace detection { 15 | // 16 | // detection 17 | Detection::Detection(cv::Mat f, 18 | const std::vector &b) 19 | : frame(f), bboxes(b) {} 20 | 21 | // Impl definition 22 | struct Yolo::Impl { 23 | Impl() { 24 | auto temp = 25 | std::make_unique(Task::DETECTION); 26 | model = std::move(temp); 27 | } 28 | ~Impl() = default; 29 | 30 | std::unique_ptr model{nullptr}; 31 | }; 32 | 33 | Yolo::Yolo() { 34 | auto impl_p = std::make_unique(); 35 | impl = std::move(impl_p); 36 | } 37 | Yolo::~Yolo() = default; 38 | 39 | std::vector 40 | Yolo::run(std::filesystem::path video) { 41 | // 42 | std::vector detections; 43 | 44 | // 45 | std::string vid_s = video.string(); 46 | cv::VideoCapture vcapt(vid_s); 47 | while (vcapt.isOpened()) { 48 | cv::Mat frame; 49 | bool isSuccess = vcapt.read(frame); 50 | // If frames are present, show it 51 | if (isSuccess == true) { 52 | // infer with model 53 | std::vector input_t = 54 | yolo_preprocess_image(frame); 55 | torch::Tensor input = torch::from_blob( 56 | input_t.data(), 57 | {1, kYOLO_CHANNEL_, kYOLO_NETWORK_HEIGHT_, 58 | kYOLO_NETWORK_WIDTH_}, 59 | torch::kFloat32); 60 | input = input.to(torch::kCPU); 61 | std::vector inputs; 62 | inputs.push_back(input); 63 | auto [outputs, shapes] = impl->model->infer(inputs); 64 | cv::Size frame_size(frame.cols, frame.rows); 65 | std::vector results = 66 | detection::yolo_postprocess(outputs, shapes, 67 | frame_size); 68 | Detection d(frame, results); 69 | detections.push_back(d); 70 | } 71 | 72 | // If frames are not there, close it 73 | if (isSuccess == false) { 74 | break; 75 | } 76 | } 77 | return detections; 78 | } 79 | } // namespace detection 80 | 81 | }; // namespace mlsamples 82 | -------------------------------------------------------------------------------- /cpp/src/run.cpp: -------------------------------------------------------------------------------- 1 | // main entry point 2 | #include 3 | // 4 | #include 5 | 6 | // 7 | #include 8 | #include 9 | 10 | namespace mlsamples { 11 | void run_task(Backend b, Task t, 12 | std::filesystem::path in_video, 13 | std::filesystem::path out) { 14 | Pipeline pipe(t, b, out); 15 | pipe.run(in_video); 16 | } 17 | 18 | } // namespace mlsamples 19 | 20 | int main(int argc, char *argv[]) { 21 | // 22 | argparse::ArgumentParser parser("ai task runner"); 23 | parser.add_argument("--backend") 24 | .default_value("yolo") 25 | .choices("yolo"); 26 | parser.add_argument("--task") 27 | .default_value("segment") 28 | .choices("segment", "detect", "pose") 29 | .required(); 30 | parser.add_argument("--video") 31 | .help("path to input video") 32 | .required(); 33 | 34 | parser.add_argument("--save_name") 35 | .help("save name for output video, e.g. out.mp4") 36 | .default_value("out.mp4") 37 | .required(); 38 | try { 39 | parser.parse_args(argc, argv); 40 | } catch (const std::exception &err) { 41 | std::cerr << err.what() << std::endl; 42 | std::cerr << parser; 43 | std::exit(1); 44 | } 45 | std::string msg = "unexpected argument for "; 46 | // 47 | std::string back = parser.get("--backend"); 48 | if (back != std::string("yolo")) { 49 | std::string m = msg + "--backend "; 50 | m += back; 51 | throw std::runtime_error(m.c_str()); 52 | } 53 | mlsamples::Backend b = mlsamples::Backend::YOLO; 54 | // 55 | mlsamples::Task t = mlsamples::Task::SEGMENTATION; 56 | std::string p_t = parser.get("--task"); 57 | if (p_t == std::string("segment")) { 58 | t = mlsamples::Task::SEGMENTATION; 59 | } else if (p_t == std::string("detect")) { 60 | t = mlsamples::Task::DETECTION; 61 | } else if (p_t == std::string("pose")) { 62 | t = mlsamples::Task::POSE_ESTIMATION; 63 | } else { 64 | std::string m = msg + "--task "; 65 | m += p_t; 66 | throw std::runtime_error(m); 67 | } 68 | // 69 | std::string in_v = parser.get("--video"); 70 | std::filesystem::path in_video(in_v); 71 | in_video = 72 | std::filesystem::absolute(in_video).make_preferred(); 73 | if (!std::filesystem::exists(in_video)) { 74 | std::string m("argument to --video "); 75 | m += in_v; 76 | m += " doesn't exist"; 77 | throw std::runtime_error(m.c_str()); 78 | } 79 | 80 | std::string s_v = parser.get("--save_name"); 81 | std::filesystem::path out_v(s_v); 82 | out_v = std::filesystem::absolute(out_v).make_preferred(); 83 | mlsamples::run_task(b, t, in_video, out_v); 84 | std::cout << "all done" << std::endl; 85 | return 0; 86 | } 87 | -------------------------------------------------------------------------------- /python/src/mlsamples/misc/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | \brief basic utility functions 3 | """ 4 | from typing import Any, Optional 5 | from uuid import uuid4 6 | from enum import Enum, auto 7 | 8 | from ultralytics import YOLO 9 | import detectron2 10 | from detectron2 import model_zoo 11 | from detectron2.engine import DefaultPredictor 12 | from detectron2.config import get_cfg 13 | import torch 14 | import numpy as np 15 | 16 | 17 | def is_type(field_value, field_name: str, field_type, raise_error: bool = True) -> bool: 18 | "check type of field value" 19 | if not isinstance(field_name, str): 20 | raise TypeError( 21 | "field_name {0} must be a string but it has type {1}".format( 22 | str(field_name), str(type(field_name)) 23 | ) 24 | ) 25 | if not isinstance(field_value, field_type): 26 | if raise_error: 27 | raise TypeError( 28 | "field_value {0} must be a {1} but it has type {2}".format( 29 | str(field_value), str(field_type), str(type(field_value)) 30 | ) 31 | ) 32 | return False 33 | return True 34 | 35 | 36 | def is_optional_type( 37 | field_value, field_name: str, field_type, raise_error: bool = True 38 | ) -> bool: 39 | "check type of field value" 40 | if field_value is None: 41 | return True 42 | else: 43 | return is_type(field_value, field_name, field_type, raise_error) 44 | 45 | 46 | class FrameContainer: 47 | """""" 48 | 49 | def __init__(self, frame: Optional[torch.Tensor] = None): 50 | is_optional_type(frame, "frame", torch.Tensor, True) 51 | self._frame = frame 52 | 53 | def set_frame(self, f: torch.Tensor): 54 | """""" 55 | is_type(f, "f", torch.Tensor, True) 56 | self._frame = f 57 | 58 | @property 59 | def frame(self) -> torch.Tensor: 60 | if self._frame is None: 61 | raise ValueError("frame is none") 62 | return self._frame 63 | 64 | 65 | class Task(Enum): 66 | SEGMENTATION = auto() 67 | DETECTION = auto() 68 | POSE_ESTIMATION = auto() 69 | 70 | 71 | def load_yolo(model_type: Task) -> YOLO: 72 | """ 73 | Load yolo model 74 | """ 75 | is_type(model_type, "model_type", Task, True) 76 | if model_type == Task.SEGMENTATION: 77 | return YOLO("yolov8x-seg.pt") 78 | elif model_type == Task.DETECTION: 79 | return YOLO("yolov8x.pt") 80 | elif model_type == Task.POSE_ESTIMATION: 81 | return YOLO("yolov8x-pose.pt") 82 | else: 83 | raise ValueError(f"unknown task {str(model_type)}") 84 | 85 | 86 | def load_detectron(task: Task): 87 | """ 88 | load detectron model 89 | """ 90 | cfg = get_cfg() 91 | if task == task.SEGMENTATION: 92 | path = "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml" 93 | elif task == task.DETECTION: 94 | path = "COCO-Detection/faster_rcnn_R_101_FPN_3x.yaml" 95 | elif task == task.POSE_ESTIMATION: 96 | path = "COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml" 97 | else: 98 | raise ValueError(f"unknown task {str(model_type)}") 99 | 100 | cfg.merge_from_file(model_zoo.get_config_file(path)) 101 | cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 102 | cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(path) 103 | cfg.MODEL.DEVICE = "cpu" 104 | model = DefaultPredictor(cfg) 105 | return model 106 | 107 | 108 | class Backend(Enum): 109 | YOLO = auto() 110 | DETECTRON = auto() 111 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | venv_samples/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # pytype static type analyzer 151 | .pytype/ 152 | 153 | # Cython debug symbols 154 | cython_debug/ 155 | 156 | # PyCharm 157 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 158 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 159 | # and can be added to the global gitignore or merged into this file. For a more nuclear 160 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 161 | #.idea/ 162 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/task/segmentation.cpp: -------------------------------------------------------------------------------- 1 | // implement detection related stuff 2 | #include "yoloutils.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | namespace mlsamples { 14 | namespace segmentation { 15 | // 16 | // segmentation 17 | Mask::Mask( 18 | cv::Mat f, 19 | const std::vector>> &m) 20 | : frame(f), masks(m) {} 21 | 22 | // Impl definition 23 | struct Yolo::Impl { 24 | Impl() { 25 | auto temp = 26 | std::make_unique(Task::SEGMENTATION); 27 | model = std::move(temp); 28 | } 29 | ~Impl() = default; 30 | std::unique_ptr model; 31 | }; 32 | 33 | Yolo::Yolo() { 34 | auto impl_p = std::make_unique(); 35 | impl = std::move(impl_p); 36 | } 37 | Yolo::~Yolo() = default; 38 | 39 | std::vector>> 40 | to_mask_points(const SegOutput &results, cv::Mat frame) { 41 | cv::Mat masks; 42 | std::vector mask_channels; 43 | if (!results.bboxes.empty()) { 44 | cv::Mat roi_m = results.mask_proposals * results.segm; 45 | cv::Mat roi_t = roi_m.t(); 46 | masks = 47 | roi_t.reshape(results.bboxes.size(), {160, 160}); 48 | cv::split(masks, mask_channels); 49 | } 50 | // 51 | std::vector>> 52 | frame_points; 53 | for (int i = 0; i < results.bboxes.size(); ++i) { 54 | std::vector> mask_points; 55 | cv::Mat temp; 56 | cv::exp(-mask_channels[i], temp); 57 | 58 | // Sigmoid 59 | cv::Mat mask = 1.0 / (1.0 + temp); 60 | 61 | mask = mask(results.roi); 62 | cv::resize(mask, mask, cv::Size(frame.cols, frame.rows), 63 | cv::INTER_NEAREST); 64 | cv::Mat m3 = mask(results.bboxes[i]); 65 | const float mask_threshold = 0.025f; 66 | 67 | for (int w = 0; w < results.bboxes[i].width; ++w) { 68 | for (int h = 0; h < results.bboxes[i].height; ++h) { 69 | int x2 = results.bboxes[i].x + w; 70 | int y2 = results.bboxes[i].y + h; 71 | float pix = m3.at(cv::Point(w, h)); 72 | if (pix > mask_threshold) { 73 | std::pair point = 74 | std::make_pair(x2, y2); 75 | mask_points.push_back(point); 76 | } 77 | } 78 | } 79 | // std::sort(mask_points.begin(), mask_points.end()); 80 | frame_points.push_back(mask_points); 81 | } 82 | return frame_points; 83 | } 84 | 85 | std::vector Yolo::run(std::filesystem::path video) { 86 | // 87 | std::vector masks; 88 | 89 | // 90 | std::string vid_s = video.string(); 91 | cv::VideoCapture vcapt(vid_s); 92 | while (vcapt.isOpened()) { 93 | cv::Mat frame; 94 | bool isSuccess = vcapt.read(frame); 95 | // If frames are present, show it 96 | if (isSuccess == true) { 97 | // infer with model 98 | std::vector input_t = 99 | yolo_preprocess_image(frame); 100 | torch::Tensor input = torch::from_blob( 101 | input_t.data(), 102 | {1, kYOLO_CHANNEL_, kYOLO_NETWORK_HEIGHT_, 103 | kYOLO_NETWORK_WIDTH_}, 104 | torch::kFloat32); 105 | input = input.to(torch::kCPU); 106 | std::vector inputs; 107 | inputs.push_back(input); 108 | auto [outputs, shapes] = impl->model->infer(inputs); 109 | cv::Size frame_size(frame.cols, frame.rows); 110 | SegOutput results = 111 | yolo_postprocess(outputs, shapes, frame_size); 112 | std::vector>> points = 113 | to_mask_points(results, frame); 114 | Mask d(frame, points); 115 | masks.push_back(d); 116 | } 117 | 118 | // If frames are not there, close it 119 | if (isSuccess == false) { 120 | break; 121 | } 122 | } 123 | return masks; 124 | } 125 | } // namespace segmentation 126 | 127 | }; // namespace mlsamples 128 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==2.0.0 2 | action-msgs==1.6.0 3 | action-tutorials-interfaces==0.27.1 4 | action-tutorials-py==0.27.1 5 | actionlib-msgs==5.0.0 6 | ament-cmake-test==2.0.3 7 | ament-copyright==0.14.2 8 | ament-cppcheck==0.14.2 9 | ament-cpplint==0.14.2 10 | ament-flake8==0.14.2 11 | ament-index-python==1.5.2 12 | ament-lint==0.14.2 13 | ament-lint-cmake==0.14.2 14 | ament-package==0.15.3 15 | ament-pep257==0.14.2 16 | ament-uncrustify==0.14.2 17 | ament-xmllint==0.14.2 18 | angles==1.16.0 19 | antlr4-python3-runtime==4.9.3 20 | av==11.0.0 21 | black==23.11.0 22 | builtin-interfaces==1.6.0 23 | cachetools==5.3.2 24 | certifi==2023.11.17 25 | charset-normalizer==3.3.2 26 | click==8.1.7 27 | cloudpickle==3.0.0 28 | composition-interfaces==1.6.0 29 | contourpy==1.2.0 30 | cv-bridge==3.4.0 31 | cycler==0.12.1 32 | decorator==4.4.2 33 | demo-nodes-py==0.27.1 34 | detectron2==0.6 35 | diagnostic-msgs==5.0.0 36 | domain-coordinator==0.11.2 37 | example-interfaces==0.10.2 38 | examples-rclpy-executors==0.18.0 39 | examples-rclpy-minimal-action-client==0.18.0 40 | examples-rclpy-minimal-action-server==0.18.0 41 | examples-rclpy-minimal-client==0.18.0 42 | examples-rclpy-minimal-publisher==0.18.0 43 | examples-rclpy-minimal-service==0.18.0 44 | examples-rclpy-minimal-subscriber==0.18.0 45 | filelock==3.13.1 46 | fonttools==4.46.0 47 | fsspec==2023.12.1 48 | fvcore==0.1.5.post20221221 49 | geometry-msgs==5.0.0 50 | google-auth==2.25.2 51 | google-auth-oauthlib==1.1.0 52 | grpcio==1.60.0 53 | hydra-core==1.3.2 54 | idna==3.6 55 | image-geometry==3.4.0 56 | imageio==2.33.0 57 | imageio-ffmpeg==0.4.9 58 | interactive-markers==2.4.0 59 | iopath==0.1.9 60 | Jinja2==3.1.2 61 | kiwisolver==1.4.5 62 | laser-geometry==2.5.0 63 | launch==2.0.2 64 | launch-ros==0.24.0 65 | launch-testing==2.0.2 66 | launch-testing-ros==0.24.0 67 | launch-xml==2.0.2 68 | launch-yaml==2.0.2 69 | lifecycle-msgs==1.6.0 70 | logging-demo==0.27.1 71 | map-msgs==2.2.0 72 | Markdown==3.5.1 73 | MarkupSafe==2.1.3 74 | matplotlib==3.8.2 75 | message-filters==4.7.0 76 | -e git+ssh://git@github.com/D-K-E/mlsamples.git@ffc030afb9e9caa2d2e743211e1e0a12e6ca8850#egg=mlsamples&subdirectory=python 77 | moviepy==1.0.3 78 | mpmath==1.3.0 79 | mypy-extensions==1.0.0 80 | nav-msgs==5.0.0 81 | networkx==3.2.1 82 | numpy==1.26.2 83 | nvidia-cublas-cu12==12.1.3.1 84 | nvidia-cuda-cupti-cu12==12.1.105 85 | nvidia-cuda-nvrtc-cu12==12.1.105 86 | nvidia-cuda-runtime-cu12==12.1.105 87 | nvidia-cudnn-cu12==8.9.2.26 88 | nvidia-cufft-cu12==11.0.2.54 89 | nvidia-curand-cu12==10.3.2.106 90 | nvidia-cusolver-cu12==11.4.5.107 91 | nvidia-cusparse-cu12==12.1.0.106 92 | nvidia-nccl-cu12==2.18.1 93 | nvidia-nvjitlink-cu12==12.3.101 94 | nvidia-nvtx-cu12==12.1.105 95 | oauthlib==3.2.2 96 | omegaconf==2.3.0 97 | opencv-python==4.8.1.78 98 | osrf-pycommon==2.1.2 99 | packaging==23.2 100 | pandas==2.1.4 101 | pathspec==0.11.2 102 | pcl-msgs==1.0.0 103 | pendulum-msgs==0.27.1 104 | Pillow==10.1.0 105 | platformdirs==4.1.0 106 | portalocker==2.8.2 107 | proglog==0.1.10 108 | protobuf==4.23.4 109 | psutil==5.9.6 110 | py-cpuinfo==9.0.0 111 | pyasn1==0.5.1 112 | pyasn1-modules==0.3.0 113 | pycocotools==2.0.7 114 | pyparsing==3.1.1 115 | python-dateutil==2.8.2 116 | python-qt-binding==1.2.3 117 | pytz==2023.3.post1 118 | PyYAML==6.0.1 119 | qt-dotgraph==2.4.2 120 | qt-gui==2.4.2 121 | qt-gui-cpp==2.4.2 122 | qt-gui-py-common==2.4.2 123 | quality-of-service-demo-py==0.27.1 124 | rcl-interfaces==1.6.0 125 | rclpy==4.1.3 126 | rcutils==6.2.1 127 | requests==2.31.0 128 | requests-oauthlib==1.3.1 129 | resource-retriever==3.2.2 130 | rmw-dds-common==2.0.1 131 | ros2action==0.25.3 132 | ros2bag==0.22.4 133 | ros2bag-mcap-cli==0.22.4 134 | ros2bag-sqlite3-cli==0.22.4 135 | ros2cli==0.25.3 136 | ros2component==0.25.3 137 | ros2doctor==0.25.3 138 | ros2interface==0.25.3 139 | ros2launch==0.24.0 140 | ros2lifecycle==0.25.3 141 | ros2multicast==0.25.3 142 | ros2node==0.25.3 143 | ros2param==0.25.3 144 | ros2pkg==0.25.3 145 | ros2run==0.25.3 146 | ros2service==0.25.3 147 | ros2topic==0.25.3 148 | rosbag2-interfaces==0.22.4 149 | rosbag2-py==0.22.4 150 | rosgraph-msgs==1.6.0 151 | rosidl-adapter==4.0.1 152 | rosidl-cli==4.0.1 153 | rosidl-cmake==4.0.1 154 | rosidl-generator-c==4.0.1 155 | rosidl-generator-cpp==4.0.1 156 | rosidl-generator-py==0.18.0 157 | rosidl-generator-type-description==4.0.1 158 | rosidl-parser==4.0.1 159 | rosidl-pycommon==4.0.1 160 | rosidl-runtime-py==0.12.0 161 | rosidl-typesupport-c==3.0.1 162 | rosidl-typesupport-cpp==3.0.1 163 | rosidl-typesupport-fastrtps-c==3.0.1 164 | rosidl-typesupport-fastrtps-cpp==3.0.1 165 | rosidl-typesupport-introspection-c==4.0.1 166 | rosidl-typesupport-introspection-cpp==4.0.1 167 | rpyutils==0.3.2 168 | rqt-action==2.1.2 169 | rqt-bag==1.3.4 170 | rqt-bag-plugins==1.3.4 171 | rqt-console==2.1.1 172 | rqt-graph==1.4.2 173 | rqt-gui==1.3.3 174 | rqt-gui-py==1.3.3 175 | rqt-msg==1.3.1 176 | rqt-plot==1.2.3 177 | rqt-publisher==1.6.3 178 | rqt-py-common==1.3.3 179 | rqt-py-console==1.1.1 180 | rqt-reconfigure==1.3.3 181 | rqt-service-caller==1.1.1 182 | rqt-shell==1.1.1 183 | rqt-srv==1.1.1 184 | rqt-topic==1.6.1 185 | rsa==4.9 186 | scipy==1.11.4 187 | seaborn==0.13.0 188 | sensor-msgs==5.0.0 189 | sensor-msgs-py==5.0.0 190 | service-msgs==1.6.0 191 | shape-msgs==5.0.0 192 | six==1.16.0 193 | sros2==0.11.3 194 | statistics-msgs==1.6.0 195 | std-msgs==5.0.0 196 | std-srvs==5.0.0 197 | stereo-msgs==5.0.0 198 | sympy==1.12 199 | tabulate==0.9.0 200 | teleop-twist-keyboard==2.3.2 201 | tensorboard==2.15.1 202 | tensorboard-data-server==0.7.2 203 | termcolor==2.4.0 204 | tf2-geometry-msgs==0.31.5 205 | tf2-kdl==0.31.5 206 | tf2-msgs==0.31.5 207 | tf2-py==0.31.5 208 | tf2-ros-py==0.31.5 209 | tf2-sensor-msgs==0.31.5 210 | tf2-tools==0.31.5 211 | thop==0.1.1.post2209072238 212 | tomli==2.0.1 213 | topic-monitor==0.27.1 214 | torch==2.1.1 215 | torchvision==0.16.1 216 | tqdm==4.66.1 217 | trajectory-msgs==5.0.0 218 | triton==2.1.0 219 | turtlesim==1.6.1 220 | type-description-interfaces==1.6.0 221 | typing_extensions==4.8.0 222 | tzdata==2023.3 223 | ultralytics==8.0.225 224 | unique-identifier-msgs==2.3.2 225 | urllib3==2.1.0 226 | visualization-msgs==5.0.0 227 | Werkzeug==3.0.1 228 | yacs==0.1.8 229 | -------------------------------------------------------------------------------- /python/src/mlsamples/task/segmentation/sgdepth.py: -------------------------------------------------------------------------------- 1 | """ 2 | sgdepth 3 | """ 4 | 5 | from mlsamples.task.segmentation.segment_interface import SegmentationMask, Segmenter 6 | from collections.abc import Iterator 7 | from libs.SGDepth.models.sgdepth import SGDepth 8 | import torch 9 | from torchvision.io import read_video 10 | import torchvision.transforms as transforms 11 | 12 | 13 | class SGDepthMask(SegmentationMask): 14 | """""" 15 | 16 | def __init__(self, result: dict): 17 | frame = result.frame 18 | depth = result.depth_img 19 | segment = result.segment_img 20 | if isinstance(frame, np.ndarray): 21 | frame = torch.tensor(frame) 22 | points = result.masks.xy 23 | masks = [] 24 | for point_arr in points: 25 | ps = [] 26 | for point in point_arr: 27 | x = int(point[0]) 28 | y = int(point[1]) 29 | ps.append((x, y)) 30 | masks.append(ps) 31 | super().__init__(masks=masks, frame=frame) 32 | 33 | 34 | class SGDepthSegmenter(Segmenter): 35 | """Inference without harness or dataloader""" 36 | 37 | def __init__(self, model: Path): 38 | self.model_path = model 39 | self.num_classes = 20 40 | self.depth_min = 0 41 | self.depth_max = 10 42 | self.all_time = [] 43 | # try: 44 | # self.checkpoint_path = os.environ['IFN_DIR_CHECKPOINT'] 45 | # except KeyError: 46 | # print('No IFN_DIR_CHECKPOINT defined.') 47 | 48 | self.labels = ( 49 | ("CLS_ROAD", (128, 64, 128)), 50 | ("CLS_SIDEWALK", (244, 35, 232)), 51 | ("CLS_BUILDING", (70, 70, 70)), 52 | ("CLS_WALL", (102, 102, 156)), 53 | ("CLS_FENCE", (190, 153, 153)), 54 | ("CLS_POLE", (153, 153, 153)), 55 | ("CLS_TRLIGHT", (250, 170, 30)), 56 | ("CLS_TRSIGN", (220, 220, 0)), 57 | ("CLS_VEGT", (107, 142, 35)), 58 | ("CLS_TERR", (152, 251, 152)), 59 | ("CLS_SKY", (70, 130, 180)), 60 | ("CLS_PERSON", (220, 20, 60)), 61 | ("CLS_RIDER", (255, 0, 0)), 62 | ("CLS_CAR", (0, 0, 142)), 63 | ("CLS_TRUCK", (0, 0, 70)), 64 | ("CLS_BUS", (0, 60, 100)), 65 | ("CLS_TRAIN", (0, 80, 100)), 66 | ("CLS_MCYCLE", (0, 0, 230)), 67 | ("CLS_BCYCLE", (119, 11, 32)), 68 | ) 69 | self.model = None 70 | self.inference_resize_height = 192 71 | self.inference_resize_width = 640 72 | self.init_model() 73 | 74 | def segment(self, video: Path) -> Iterator[SegmentationMask]: 75 | """""" 76 | v = read_video(str(video)) 77 | clip = v[0] 78 | T, Height, Width, Channel = clip.shape 79 | for t in range(T): 80 | # frame = clip[t, :].numpy() 81 | frame = clip[t, :] 82 | batch = self.prep_batch_from(img=frame) 83 | with torch.no_grad(): 84 | output = self.model(batch) # forward pictures 85 | seg_img, depth_img = self.prep_output(output) 86 | depth_img = cv2.resize(depth_img, (Width, Height)) 87 | seg_img = cv2.resize( 88 | seg_img, 89 | (Width, Height), 90 | interpolation=cv2.INTER_NEAREST, 91 | ) 92 | orig_img = frame.numpy() 93 | result = dict(frame=orig_img, depth_img=depth_img, segment_img=seg_img) 94 | mask = SGDepthMask(result) 95 | yield mask 96 | 97 | def init_model(self): 98 | print("Init Model...") 99 | num_layers = 18 100 | split_pos = 1 101 | depth_grad_scale = 0.9 102 | segmentation_grad_scale = 0.1 103 | train_weights_init = "pretrained" 104 | depth_resolutions = 4 105 | num_layers_pose = 18 106 | 107 | with torch.no_grad(): 108 | # init 'empty' model 109 | self.model = SGDepth( 110 | split_pos, 111 | num_layers, 112 | depth_grad_scale, 113 | segmentation_grad_scale, 114 | # opt.train_domain_grad_scale, 115 | train_weights_init, 116 | depth_resolutions, 117 | num_layers_pose, 118 | # opt.model_num_domains, 119 | # opt.train_loss_weighting_strategy, 120 | # opt.train_grad_scale_weighting_strategy, 121 | # opt.train_gradnorm_alpha, 122 | # opt.train_uncertainty_eta_depth, 123 | # opt.train_uncertainty_eta_seg, 124 | # opt.model_shared_encoder_batchnorm_momentum 125 | ) 126 | 127 | # load weights (copied from state manager) 128 | state = self.model.state_dict() 129 | to_load = torch.load(self.model_path) 130 | for (k, v) in to_load.items(): 131 | if k not in state: 132 | print( 133 | f" - WARNING: Model file contains unknown key {k} ({list(v.shape)})" 134 | ) 135 | 136 | for (k, v) in state.items(): 137 | if k not in to_load: 138 | print( 139 | f" - WARNING: Model file does not contain key {k} ({list(v.shape)})" 140 | ) 141 | 142 | else: 143 | state[k] = to_load[k] 144 | 145 | self.model.load_state_dict(state) 146 | self.model = ( 147 | self.model.eval().cpu() 148 | ) # for inference model should be in eval mode and on gpu 149 | 150 | def prep_batch_from(self, img): 151 | """""" 152 | resize = transforms.Resize( 153 | (self.inference_resize_height, self.inference_resize_width) 154 | ) 155 | image = resize(img) # resize to argument size 156 | 157 | to_tensor = transforms.ToTensor() # transform to tensor 158 | 159 | input_image = to_tensor( 160 | image 161 | ) # save tensor image to self.input_image for saving later 162 | image = self.normalize(input_image) 163 | 164 | image = image.unsqueeze(0).float().cpu() 165 | 166 | # simulate structure of batch: 167 | image_dict = {("color_aug", 0, 0): image} # dict 168 | image_dict[("color", 0, 0)] = image 169 | image_dict["domain"] = [ 170 | "cityscapes_val_seg", 171 | ] 172 | image_dict["purposes"] = [ 173 | [ 174 | "segmentation", 175 | ], 176 | [ 177 | "depth", 178 | ], 179 | ] 180 | image_dict["num_classes"] = torch.tensor([self.num_classes]) 181 | image_dict["domain_idx"] = torch.tensor(0) 182 | batch = (image_dict,) # batch tuple 183 | return batch 184 | 185 | def prep_output(self, output): 186 | """""" 187 | seg_img = self.prep_seg_img(output) 188 | depth_img = self.prep_depth_img(output) 189 | return seg_img, depth_img 190 | 191 | def prep_seg_img(self, output): 192 | """""" 193 | segs_pred = output[0]["segmentation_logits", 0] # seg results 194 | 195 | segs_pred = segs_pred.exp().cpu() 196 | segs_pred = segs_pred.numpy() # transform preds to np array 197 | segs_pred = segs_pred.argmax(1) # get the highest score for classes per pixel 198 | segs_pred = segs_pred[0] 199 | o_size = segs_pred.shape 200 | 201 | # init of seg image 202 | seg_img_array = np.zeros((3, segs_pred.shape[0], segs_pred.shape[1])) 203 | 204 | # create a color image from the classes for every pixel todo: probably a lot faster if vectorized with numpy 205 | i = 0 206 | while i < segs_pred.shape[0]: # for row 207 | n = 0 208 | while n < segs_pred.shape[1]: # for column 209 | lab = 0 210 | while lab < self.num_classes: # for classes 211 | if segs_pred[i, n] == lab: 212 | # write colors to pixel 213 | seg_img_array[0, i, n] = self.labels[lab][1][0] 214 | seg_img_array[1, i, n] = self.labels[lab][1][1] 215 | seg_img_array[2, i, n] = self.labels[lab][1][2] 216 | break 217 | lab += 1 218 | n += 1 219 | i += 1 220 | 221 | # scale the color values to 0-1 for proper visualization of OpenCV 222 | seg_img = seg_img_array.transpose(1, 2, 0).astype(np.uint8) 223 | seg_img = seg_img[:, :, ::-1] 224 | return seg_img 225 | 226 | def prep_depth_img(self, output): 227 | """""" 228 | disps_pred = output[0]["disp", 0] # depth results 229 | # Depth Visualization 230 | depth_pred = np.array( 231 | disps_pred[0][0].cpu() 232 | ) # depth predictions to numpy and CPU 233 | 234 | depth_pred = self.scale_depth(depth_pred) # Depthmap in meters 235 | depth_pred = depth_pred * ( 236 | 255 / depth_pred.max() 237 | ) # Normalize Depth to 255 = max depth 238 | depth_pred = np.clip(depth_pred, 0, 255) # Clip to 255 for safety 239 | depth_pred = depth_pred.astype(np.uint8) # Cast to uint8 for openCV to display 240 | return depth_pred 241 | 242 | def normalize(self, tensor): 243 | """""" 244 | mean = (0.485, 0.456, 0.406) 245 | std = (0.229, 0.224, 0.225) 246 | 247 | normalize = transforms.Normalize(mean, std) 248 | tensor = normalize(tensor) 249 | 250 | return tensor 251 | 252 | def scale_depth(self, disp): 253 | """""" 254 | min_disp = 1 / self.depth_max 255 | max_disp = 1 / self.depth_min 256 | return min_disp + (max_disp - min_disp) * disp 257 | -------------------------------------------------------------------------------- /cpp/src/mlsamples/task/yoloutils.cpp: -------------------------------------------------------------------------------- 1 | #include "yoloutils.h" 2 | #include 3 | #include 4 | #include 5 | // implementation file 6 | 7 | namespace mlsamples { 8 | // 9 | std::vector 10 | yolo_preprocess_image(const cv::Mat &image) { 11 | cv::Mat blob; 12 | cv::cvtColor(image, blob, cv::COLOR_BGR2RGB); 13 | int target_width, target_height, offset_x, offset_y; 14 | float resize_ratio_width = 15 | static_cast(kYOLO_NETWORK_WIDTH_) / 16 | static_cast(image.cols); 17 | float resize_ratio_height = 18 | static_cast(kYOLO_NETWORK_HEIGHT_) / 19 | static_cast(image.rows); 20 | 21 | if (resize_ratio_height > resize_ratio_width) { 22 | target_width = kYOLO_NETWORK_WIDTH_; 23 | target_height = resize_ratio_width * image.rows; 24 | offset_x = 0; 25 | offset_y = (kYOLO_NETWORK_HEIGHT_ - target_height) / 2; 26 | } else { 27 | target_width = resize_ratio_height * image.cols; 28 | target_height = kYOLO_NETWORK_HEIGHT_; 29 | offset_x = (kYOLO_NETWORK_WIDTH_ - target_width) / 2; 30 | offset_y = 0; 31 | } 32 | 33 | cv::Mat resized_image(target_height, target_width, 34 | CV_8UC3); 35 | cv::resize(blob, resized_image, resized_image.size(), 0, 36 | 0, cv::INTER_LINEAR); 37 | cv::Mat output_image(kYOLO_NETWORK_WIDTH_, 38 | kYOLO_NETWORK_HEIGHT_, CV_8UC3, 39 | cv::Scalar(128, 128, 128)); 40 | resized_image.copyTo(output_image( 41 | cv::Rect(offset_x, offset_y, resized_image.cols, 42 | resized_image.rows))); 43 | output_image.convertTo(output_image, CV_32FC3, 44 | 1.f / 255.f); 45 | 46 | size_t img_byte_size = 47 | output_image.total() * 48 | output_image.elemSize(); // Allocate a buffer to hold 49 | // all image elements. 50 | std::vector input_data = std::vector( 51 | kYOLO_NETWORK_WIDTH_ * kYOLO_NETWORK_HEIGHT_ * 52 | kYOLO_CHANNEL_); 53 | std::memcpy(input_data.data(), output_image.data, 54 | img_byte_size); 55 | 56 | std::vector chw; 57 | for (size_t i = 0; i < kYOLO_CHANNEL_; ++i) { 58 | chw.emplace_back( 59 | cv::Mat(cv::Size(kYOLO_NETWORK_WIDTH_, 60 | kYOLO_NETWORK_HEIGHT_), 61 | CV_32FC1, 62 | &(input_data[i * kYOLO_NETWORK_WIDTH_ * 63 | kYOLO_NETWORK_HEIGHT_]))); 64 | } 65 | cv::split(output_image, chw); 66 | 67 | return input_data; 68 | } 69 | 70 | cv::Rect yolo_get_rect(const cv::Size &imgSz, 71 | const std::vector &bbox) { 72 | float r_w = kYOLO_NETWORK_WIDTH_ / 73 | static_cast(imgSz.width); 74 | float r_h = kYOLO_NETWORK_HEIGHT_ / 75 | static_cast(imgSz.height); 76 | 77 | int l, r, t, b; 78 | if (r_h > r_w) { 79 | l = bbox[0] - bbox[2] / 2.f; 80 | r = bbox[0] + bbox[2] / 2.f; 81 | t = bbox[1] - bbox[3] / 2.f - 82 | (kYOLO_NETWORK_HEIGHT_ - r_w * imgSz.height) / 2; 83 | b = bbox[1] + bbox[3] / 2.f - 84 | (kYOLO_NETWORK_HEIGHT_ - r_w * imgSz.height) / 2; 85 | l /= r_w; 86 | r /= r_w; 87 | t /= r_w; 88 | b /= r_w; 89 | } else { 90 | l = bbox[0] - bbox[2] / 2.f - 91 | (kYOLO_NETWORK_WIDTH_ - r_h * imgSz.width) / 2; 92 | r = bbox[0] + bbox[2] / 2.f - 93 | (kYOLO_NETWORK_WIDTH_ - r_h * imgSz.width) / 2; 94 | t = bbox[1] - bbox[3] / 2.f; 95 | b = bbox[1] + bbox[3] / 2.f; 96 | l /= r_h; 97 | r /= r_h; 98 | t /= r_h; 99 | b /= r_h; 100 | } 101 | 102 | // Clamp the coordinates within the image bounds 103 | l = std::max(0, std::min(l, imgSz.width - 1)); 104 | r = std::max(0, std::min(r, imgSz.width - 1)); 105 | t = std::max(0, std::min(t, imgSz.height - 1)); 106 | b = std::max(0, std::min(b, imgSz.height - 1)); 107 | 108 | return cv::Rect(l, t, r - l, b - t); 109 | } 110 | 111 | namespace detection { 112 | std::vector yolo_postprocess( 113 | const std::vector> &outputs, 114 | const std::vector> &shapes, 115 | const cv::Size &frame_size) { 116 | const float *output0 = outputs.front().data(); 117 | const std::vector shape0 = shapes.front(); 118 | 119 | const auto offset = 4; 120 | const auto num_classes = shape0[1] - offset; 121 | 122 | std::vector out_d(outputs[0].begin(), 123 | outputs[0].end()); 124 | cv::Mat outm(shape0[1], shape0[2], CV_32FC1, 125 | out_d.data()); 126 | cv::Mat tout_m = outm.t(); 127 | 128 | std::vector boxes; 129 | std::vector confs; 130 | 131 | // Get all the YOLO proposals 132 | for (int i = 0; i < shape0[2]; ++i) { 133 | cv::Mat trow = tout_m.row(i); 134 | cv::Mat scores_t = trow.colRange(4, trow.cols); 135 | cv::Mat box_m = trow.colRange(0, 4); 136 | auto score_it = std::max_element( 137 | scores_t.begin(), scores_t.end()); 138 | float m_score = *score_it; 139 | if (m_score > kYOLO_CONFIDENCE_THRESHOLD) { 140 | std::vector bdata(box_m.begin(), 141 | box_m.end()); 142 | cv::Rect y_rect = yolo_get_rect(frame_size, bdata); 143 | boxes.emplace_back(y_rect); 144 | confs.emplace_back(m_score); 145 | } 146 | } 147 | 148 | // Perform Non Maximum Suppression and draw predictions. 149 | std::vector indices; 150 | cv::dnn::NMSBoxes(boxes, confs, 151 | kYOLO_CONFIDENCE_THRESHOLD, 152 | kYOLO_NMS_THRESHOLD, indices); 153 | std::vector rects; 154 | for (int i = 0; i < indices.size(); i++) { 155 | int idx = indices[i]; 156 | cv::Rect box = boxes[idx]; 157 | rects.push_back(box); 158 | } 159 | return rects; 160 | } 161 | } // namespace detection 162 | 163 | namespace segmentation { 164 | 165 | // taken from 166 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/YoloV8.cpp 167 | cv::Rect find_pad_size(const size_t inputW, 168 | const size_t inputH, 169 | const cv::Size &inputSize) { 170 | std::vector padSize; 171 | float w, h, x, y; 172 | float r_w = inputW / (inputSize.width * 1.0); 173 | float r_h = inputH / (inputSize.height * 1.0); 174 | if (r_h > r_w) { 175 | w = inputW; 176 | h = r_w * inputSize.height; 177 | x = 0; 178 | y = (inputH - h) / 2; 179 | } else { 180 | w = r_h * inputSize.width; 181 | h = inputH; 182 | x = (inputW - w) / 2; 183 | y = 0; 184 | } 185 | return cv::Rect(x, y, w, h); 186 | } 187 | 188 | SegOutput::SegOutput(const std::vector &bs, 189 | const cv::Mat &s, const cv::Mat &p, 190 | const cv::Rect &r) 191 | : bboxes(bs), segm(s), mask_proposals(p), roi(r) {} 192 | 193 | SegOutput yolo_postprocess( 194 | const std::vector> &output, 195 | const std::vector> &shape, 196 | const cv::Size &frame_size) { 197 | const auto offset = 4; 198 | const auto num_classes = 199 | shape[0][1] - offset - shape[1][1]; 200 | 201 | // Construct output matrix 202 | std::vector mat_d(output[0].size()); 203 | std::copy(output[0].begin(), output[0].end(), 204 | mat_d.begin()); 205 | cv::Mat outm(shape[0][1], shape[0][2], CV_32FC1, 206 | mat_d.data()); 207 | 208 | // Transpose output matrix 209 | cv::Mat tout_m = outm.t(); 210 | 211 | std::vector boxes; 212 | std::vector confs; 213 | 214 | std::vector> picked_proposals; 215 | 216 | // Get all the YOLO proposals 217 | for (int i = 0; i < shape[0][2]; ++i) { 218 | cv::Mat row = tout_m.row(i); 219 | cv::Mat scores = row.colRange(4, num_classes); 220 | auto it = std::max_element(scores.begin(), 221 | scores.end()); 222 | float score = *it; 223 | if (score > kYOLO_CONFIDENCE_THRESHOLD) { 224 | cv::Mat mbox = row.colRange(0, 4); 225 | std::vector mm(mbox.begin(), 226 | mbox.end()); 227 | boxes.emplace_back(yolo_get_rect(frame_size, mm)); 228 | confs.emplace_back(score); 229 | int end = num_classes + shape[1][1]; 230 | cv::Mat prange = row.colRange(num_classes, end); 231 | std::vector proposal(prange.begin(), 232 | prange.end()); 233 | picked_proposals.emplace_back(proposal); 234 | } 235 | } 236 | 237 | // Perform Non Maximum Suppression and draw predictions. 238 | std::vector indices; 239 | cv::dnn::NMSBoxes(boxes, confs, 240 | kYOLO_CONFIDENCE_THRESHOLD, 241 | kYOLO_NMS_THRESHOLD, indices); 242 | int sc, sh, sw; 243 | std::tie(sc, sh, sw) = 244 | std::make_tuple(static_cast(shape[1][1]), 245 | static_cast(shape[1][2]), 246 | static_cast(shape[1][3])); 247 | 248 | std::vector mask_d(output[1].begin(), 249 | output[1].end()); 250 | cv::Mat segm = 251 | cv::Mat(sc, sw * sh, CV_32FC1, mask_d.data()); 252 | cv::Rect pad_rect = 253 | find_pad_size(kYOLO_NETWORK_WIDTH_, 254 | kYOLO_NETWORK_HEIGHT_, frame_size); 255 | cv::Rect roi( 256 | int((float)pad_rect.x / kYOLO_NETWORK_WIDTH_ * sw), 257 | int((float)pad_rect.y / kYOLO_NETWORK_HEIGHT_ * sh), 258 | int(sw - pad_rect.x / 2), int(sh - pad_rect.y / 2)); 259 | 260 | cv::Mat maskProposals; 261 | std::vector bboxes; 262 | for (int i = 0; i < indices.size(); i++) { 263 | int idx = indices[i]; 264 | std::vector picked = picked_proposals[idx]; 265 | cv::Rect bbox = boxes[idx]; 266 | bboxes.push_back(bbox); 267 | cv::Mat pmat = cv::Mat(picked).t(); 268 | maskProposals.push_back(pmat); 269 | } 270 | SegOutput seg(bboxes, segm, maskProposals, roi); 271 | 272 | return seg; 273 | } 274 | 275 | } // namespace segmentation 276 | 277 | YoloV8Model::YoloV8Model(Task t) { 278 | std::filesystem::path p(ASSET_DIR); 279 | p = p.make_preferred(); 280 | if (t == Task::DETECTION) { 281 | p /= std::filesystem::path("yolov8x.torchscript"); 282 | } else if (t == Task::SEGMENTATION) { 283 | p /= std::filesystem::path("yolov8x-seg.torchscript"); 284 | // p /= std::filesystem::path("yolov8x.torchscript"); 285 | } else { 286 | throw std::runtime_error("Unsupported task"); 287 | } 288 | 289 | if (!std::filesystem::exists(p)) { 290 | std::string m("model can not " 291 | "be found in: "); 292 | m += p.string(); 293 | throw std::runtime_error(m.c_str()); 294 | } 295 | model = mlsamples::load(p); 296 | model.eval(); 297 | } 298 | std::pair>, 299 | std::vector>> 300 | YoloV8Model::infer(std::vector inputs) { 301 | // from 302 | // https://github.com/olibartfast/object-detection-inference/blob/master/src/libtorch/LibtorchInfer.cpp 303 | // 304 | auto output = model.forward(inputs); 305 | std::vector> output_vectors; 306 | std::vector> shape_vectors; 307 | 308 | if (output.isTuple()) { 309 | // Handle the case where the model returns a tuple 310 | auto tuple_outputs = output.toTuple()->elements(); 311 | 312 | for (const auto &output_tensor : tuple_outputs) { 313 | if (!output_tensor.isTensor()) 314 | continue; 315 | torch::Tensor tensor = output_tensor.toTensor() 316 | .to(torch::kCPU) 317 | .contiguous(); 318 | 319 | // Get the output data as a float pointer 320 | const float *output_data = tensor.data_ptr(); 321 | 322 | // Store the output data in the outputs vector 323 | std::vector output_vector( 324 | output_data, output_data + tensor.numel()); 325 | output_vectors.push_back(output_vector); 326 | 327 | // Get the shape of the output tensor 328 | std::vector shape = tensor.sizes().vec(); 329 | shape_vectors.push_back(shape); 330 | } 331 | } else { 332 | torch::Tensor tensor = output.toTensor(); 333 | if (tensor.size(0) == 1) { 334 | // If there's only one output tensor 335 | torch::Tensor output_tensor = 336 | tensor.to(torch::kCPU).contiguous(); 337 | 338 | // Get the output data as a float pointer 339 | const float *output_data = 340 | output_tensor.data_ptr(); 341 | 342 | // Store the output data and shape in vectors 343 | output_vectors.emplace_back( 344 | output_data, output_data + output_tensor.numel()); 345 | shape_vectors.push_back(output_tensor.sizes().vec()); 346 | } else { 347 | for (int i = 0; i < tensor.size(0); ++i) { 348 | torch::Tensor output_tensor = 349 | tensor[i].to(torch::kCPU).contiguous(); 350 | 351 | // Get the output data as a float pointer 352 | const float *output_data = 353 | output_tensor.data_ptr(); 354 | 355 | // Store the output data and shape in vectors 356 | output_vectors.emplace_back( 357 | output_data, 358 | output_data + output_tensor.numel()); 359 | shape_vectors.push_back( 360 | output_tensor.sizes().vec()); 361 | } 362 | } 363 | } 364 | auto p = std::make_pair(output_vectors, shape_vectors); 365 | return p; 366 | } 367 | 368 | } // namespace mlsamples 369 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | -------------------------------------------------------------------------------- /python/LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | --------------------------------------------------------------------------------