├── movery ├── go │ └── cmd │ │ └── movery │ │ └── main.go ├── reporters │ └── __init__.py ├── config │ ├── __init__.py │ ├── config.json │ └── config.py ├── analyzers │ ├── __init__.py │ └── code_analyzer.py ├── detectors │ └── __init__.py ├── utils │ ├── __init__.py │ ├── memory.py │ └── logging.py ├── __init__.py ├── config.json ├── tests │ ├── unit │ │ ├── test_detector.py │ │ ├── test_analyzer.py │ │ ├── test_vulnerability.py │ │ └── test_security.py │ └── security │ │ └── test_security.py ├── templates │ └── report.html └── main.py ├── go ├── internal │ ├── detectors │ │ ├── tests │ │ │ └── detector_test.go │ │ └── vulnerability.go │ ├── reporters │ │ ├── json.go │ │ └── xml.go │ ├── cmd │ │ ├── web.go │ │ ├── root.go │ │ ├── server.go │ │ └── scan.go │ ├── utils │ │ ├── logging.go │ │ ├── parallel.go │ │ ├── memory.go │ │ └── security_test.go │ ├── web │ │ ├── static │ │ │ └── css │ │ │ │ └── style.css │ │ └── app.go │ ├── core │ │ ├── models.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── scanner.go │ │ └── scanner_test.go │ ├── analyzers │ │ └── language.go │ ├── config │ │ └── config.go │ └── api │ │ └── server.go ├── cmd │ └── movery │ │ └── main.go ├── go.mod ├── README.md ├── web │ └── templates │ │ └── report.html └── tests │ ├── security │ └── security_test.go │ └── integration │ └── workflow_test.go ├── config ├── ctags └── movery_config.py ├── requirements.txt ├── .gitignore ├── LICENSE ├── CODE_OF_CONDUCT.md ├── config.json.example ├── Makefile ├── config.json ├── setup.py ├── .github └── workflows │ └── go.yml ├── CONTRIBUTING.md ├── signatures.json.example ├── src ├── config │ └── config.py ├── utils │ ├── logging.py │ └── memory.py └── main.py ├── signatures.json ├── README.md └── docs └── test_report.md /movery/go/cmd/movery/main.go: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /go/internal/detectors/tests/detector_test.go: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /config/ctags: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyangxu/Re-movery/HEAD/config/ctags -------------------------------------------------------------------------------- /movery/reporters/__init__.py: -------------------------------------------------------------------------------- 1 | from .html import HTMLReporter 2 | 3 | __all__ = ['HTMLReporter'] -------------------------------------------------------------------------------- /config/movery_config.py: -------------------------------------------------------------------------------- 1 | vulpath = 'D:/NEWRESEARCH/vulFuncs/' 2 | oldpath = 'D:/NEWRESEARCH/oldestFuncs/' -------------------------------------------------------------------------------- /movery/config/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration module for Movery 3 | """ 4 | from .config import config 5 | 6 | __all__ = ['config'] -------------------------------------------------------------------------------- /movery/analyzers/__init__.py: -------------------------------------------------------------------------------- 1 | from .language import LanguageAnalyzer 2 | from .code_analyzer import CodeAnalyzer 3 | 4 | __all__ = ['LanguageAnalyzer', 'CodeAnalyzer'] -------------------------------------------------------------------------------- /movery/detectors/__init__.py: -------------------------------------------------------------------------------- 1 | from .vulnerability import VulnerabilityDetector, Signature, VulnerabilityMatch 2 | 3 | __all__ = ['VulnerabilityDetector', 'Signature', 'VulnerabilityMatch'] -------------------------------------------------------------------------------- /movery/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .security import SecurityChecker 2 | from .parallel import WorkerPool, ParallelExecutor 3 | from .logging import get_logger 4 | from .memory import MemoryMonitor 5 | 6 | __all__ = ['SecurityChecker', 'WorkerPool', 'ParallelExecutor', 'get_logger', 'MemoryMonitor'] -------------------------------------------------------------------------------- /go/cmd/movery/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/re-movery/re-movery/internal/cmd" 8 | ) 9 | 10 | func main() { 11 | // 执行根命令 12 | if err := cmd.Execute(); err != nil { 13 | fmt.Fprintf(os.Stderr, "Error: %v\n", err) 14 | os.Exit(1) 15 | } 16 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2>=3.0.0 2 | plotly>=5.0.0 3 | pandas>=1.3.0 4 | psutil>=5.8.0 5 | tqdm>=4.61.0 6 | colorama>=0.4.4 7 | requests>=2.26.0 8 | beautifulsoup4>=4.9.3 9 | lxml>=4.6.3 10 | pygments>=2.9.0 11 | typing-extensions>=3.10.0 12 | dataclasses>=0.8;python_version<"3.7" 13 | astroid>=2.15.0 14 | -------------------------------------------------------------------------------- /movery/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Re-Movery - A tool for discovering modified vulnerable code clones 3 | """ 4 | 5 | __version__ = "1.0.0" 6 | __author__ = "heyangxu" 7 | __email__ = "" 8 | 9 | from .config.config import config 10 | from .detectors.vulnerability import VulnerabilityDetector 11 | from .utils.security import SecurityChecker 12 | 13 | __all__ = ["config", "VulnerabilityDetector", "SecurityChecker"] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Go 24 | *.exe 25 | *.exe~ 26 | *.dll 27 | *.so 28 | *.dylib 29 | *.test 30 | *.out 31 | go.work 32 | /go/bin/ 33 | /go/pkg/ 34 | 35 | # IDE 36 | .idea/ 37 | .vscode/ 38 | *.swp 39 | *.swo 40 | 41 | # Project specific 42 | .cache/ 43 | reports/ 44 | *.log 45 | profile.stats 46 | .coverage 47 | htmlcov/ 48 | 49 | # Environment 50 | .env 51 | .venv 52 | env/ 53 | venv/ 54 | ENV/ 55 | 56 | # OS 57 | .DS_Store 58 | Thumbs.db 59 | 60 | # dataset 61 | dataset/ 62 | -------------------------------------------------------------------------------- /go/internal/reporters/json.go: -------------------------------------------------------------------------------- 1 | package reporters 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/re-movery/re-movery/internal/core" 9 | ) 10 | 11 | // JSONReporter is a reporter that generates JSON reports 12 | type JSONReporter struct{} 13 | 14 | // NewJSONReporter creates a new JSON reporter 15 | func NewJSONReporter() *JSONReporter { 16 | return &JSONReporter{} 17 | } 18 | 19 | // GenerateReport generates a report 20 | func (r *JSONReporter) GenerateReport(data core.ReportData, outputPath string) error { 21 | // Create output directory if it doesn't exist 22 | outputDir := filepath.Dir(outputPath) 23 | if err := os.MkdirAll(outputDir, 0755); err != nil { 24 | return err 25 | } 26 | 27 | // Create output file 28 | file, err := os.Create(outputPath) 29 | if err != nil { 30 | return err 31 | } 32 | defer file.Close() 33 | 34 | // Marshal data to JSON 35 | encoder := json.NewEncoder(file) 36 | encoder.SetIndent("", " ") 37 | if err := encoder.Encode(data); err != nil { 38 | return err 39 | } 40 | 41 | return nil 42 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 heyangxu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # 贡献者行为准则 2 | 3 | ## 我们的承诺 4 | 5 | 为了营造一个开放和友好的环境，我们作为贡献者和维护者承诺：无论年龄、体型、身体健全与否、民族、性征、性别认同和表达、经验水平、教育程度、社会地位、国籍、相貌、种族、宗教信仰、性取向如何，我们都会确保每个参与项目的人都不受骚扰。 6 | 7 | ## 我们的标准 8 | 9 | 有助于创造积极环境的行为包括： 10 | 11 | * 使用友好和包容的语言 12 | * 尊重不同的观点和经验 13 | * 优雅地接受建设性批评 14 | * 关注对社区最有利的事情 15 | * 友善对待其他社区成员 16 | 17 | 不当行为包括： 18 | 19 | * 使用带有性色彩的语言或图像，以及不受欢迎的性关注或advances 20 | * 发表挑衅、侮辱/贬损的评论，进行人身攻击或政治攻击 21 | * 公开或私下骚扰 22 | * 未经明确许可，发布他人的私人信息，如物理或电子地址 23 | * 其他可以被合理地认定为不恰当或违反职业操守的行为 24 | 25 | ## 我们的责任 26 | 27 | 项目维护者有责任为可接受的行为标准做出诠释，并采取恰当且公平的纠正措施来应对任何不可接受的行为。 28 | 29 | 项目维护者有权利和责任删除、编辑或拒绝违反本行为准则的评论、提交、代码、wiki编辑、问题和其他贡献，并暂时或永久地禁止任何他们认为不当、威胁、冒犯或有害的行为的贡献者。 30 | 31 | ## 范围 32 | 33 | 当一个人代表项目或其社区时，本行为准则适用于项目空间和公共空间。代表项目或社区的示例包括使用官方项目电子邮件地址、通过官方社交媒体账户发布，或在线上或线下活动中担任指定代表。项目的代表性可由项目维护者进一步定义和澄清。 34 | 35 | ## 强制执行 36 | 37 | 可以通过[在此处插入联系方式]向项目团队报告辱骂、骚扰或其他不可接受的行为。所有投诉都将得到审查和调查，并将导致做出适当且必要的回应。项目团队有义务对事件报告者保密。具体执行政策的更多细节可能会单独发布。 38 | 39 | 不遵守或不执行本行为准则的项目维护者可能会因项目领导层的决定而暂时或永久地失去其在项目中的角色。 40 | 41 | ## 归属 42 | 43 | 本行为准则改编自[贡献者公约][homepage]，版本1.4，可在[http://contributor-covenant.org/version/1/4][version]查看。 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ -------------------------------------------------------------------------------- /config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "processing": { 3 | "num_workers": 4, 4 | "enable_cache": true, 5 | "cache_dir": ".cache", 6 | "max_file_size_mb": 10 7 | }, 8 | "detector": { 9 | "min_similarity": 0.8, 10 | "enable_semantic_match": true, 11 | "ignore_comments": true, 12 | "ignore_whitespace": true, 13 | "max_line_distance": 100, 14 | "context_lines": 5 15 | }, 16 | "analyzer": { 17 | "languages": ["go"], 18 | "parse_comments": true, 19 | "parse_imports": true, 20 | "parse_types": true 21 | }, 22 | "reporter": { 23 | "output_format": "html", 24 | "include_source": true, 25 | "group_by_severity": true, 26 | "min_severity": "low", 27 | "template_dir": "web/templates" 28 | }, 29 | "logging": { 30 | "level": "info", 31 | "file": "movery.log", 32 | "format": "text", 33 | "include_timestamp": true 34 | }, 35 | "security": { 36 | "max_memory_gb": 8.0, 37 | "timeout_seconds": 3600, 38 | "exclude_patterns": [ 39 | "vendor/**", 40 | "node_modules/**", 41 | "**/*_test.go", 42 | "**/*.min.js" 43 | ] 44 | } 45 | } -------------------------------------------------------------------------------- /go/internal/cmd/web.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/re-movery/re-movery/internal/web" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var ( 12 | webHost string 13 | webPort int 14 | webDebug bool 15 | ) 16 | 17 | var webCmd = &cobra.Command{ 18 | Use: "web", 19 | Short: "Start the web interface", 20 | Long: `Start the web interface for Re-movery. 21 | The web interface provides a user-friendly way to scan files and directories for security vulnerabilities. 22 | 23 | Examples: 24 | re-movery web 25 | re-movery web --host 0.0.0.0 --port 8080 26 | re-movery web --debug`, 27 | Run: func(cmd *cobra.Command, args []string) { 28 | // Create web app 29 | app := web.NewApp() 30 | 31 | // Start web server 32 | addr := fmt.Sprintf("%s:%d", webHost, webPort) 33 | fmt.Printf("Starting web server at http://%s\n", addr) 34 | 35 | if err := app.Run(webHost, webPort, webDebug); err != nil { 36 | fmt.Fprintf(os.Stderr, "Error starting web server: %v\n", err) 37 | os.Exit(1) 38 | } 39 | }, 40 | } 41 | 42 | func init() { 43 | // Add flags 44 | webCmd.Flags().StringVar(&webHost, "host", "localhost", "Host to bind the web server to") 45 | webCmd.Flags().IntVar(&webPort, "port", 8080, "Port to bind the web server to") 46 | webCmd.Flags().BoolVar(&webDebug, "debug", false, "Enable debug mode") 47 | } -------------------------------------------------------------------------------- /go/internal/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var rootCmd = &cobra.Command{ 11 | Use: "re-movery", 12 | Short: "Re-movery - Security Vulnerability Scanner", 13 | Long: `Re-movery is a powerful security vulnerability scanner designed to detect 14 | potential security issues in your codebase. It supports multiple programming 15 | languages and provides various interfaces for scanning and reporting.`, 16 | Run: func(cmd *cobra.Command, args []string) { 17 | // If no subcommand is provided, print help 18 | cmd.Help() 19 | }, 20 | } 21 | 22 | // Execute executes the root command 23 | func Execute() error { 24 | return rootCmd.Execute() 25 | } 26 | 27 | func init() { 28 | // Add global flags 29 | rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose output") 30 | rootCmd.PersistentFlags().StringP("config", "c", "", "Config file path") 31 | 32 | // Add subcommands 33 | rootCmd.AddCommand(scanCmd) 34 | rootCmd.AddCommand(webCmd) 35 | rootCmd.AddCommand(serverCmd) 36 | rootCmd.AddCommand(generateCmd) 37 | rootCmd.AddCommand(versionCmd) 38 | } 39 | 40 | // versionCmd represents the version command 41 | var versionCmd = &cobra.Command{ 42 | Use: "version", 43 | Short: "Print the version number", 44 | Run: func(cmd *cobra.Command, args []string) { 45 | fmt.Println("Re-movery v1.0.0") 46 | }, 47 | } -------------------------------------------------------------------------------- /go/internal/cmd/server.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/re-movery/re-movery/internal/api" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var ( 12 | serverHost string 13 | serverPort int 14 | serverDebug bool 15 | ) 16 | 17 | var serverCmd = &cobra.Command{ 18 | Use: "server", 19 | Short: "Start the API server", 20 | Long: `Start the API server for Re-movery. 21 | The API server provides a RESTful API for scanning files and directories for security vulnerabilities. 22 | 23 | Examples: 24 | re-movery server 25 | re-movery server --host 0.0.0.0 --port 8081 26 | re-movery server --debug`, 27 | Run: func(cmd *cobra.Command, args []string) { 28 | // Create API server 29 | server := api.NewServer() 30 | 31 | // Start API server 32 | addr := fmt.Sprintf("%s:%d", serverHost, serverPort) 33 | fmt.Printf("Starting API server at http://%s\n", addr) 34 | 35 | if err := server.Run(serverHost, serverPort, serverDebug); err != nil { 36 | fmt.Fprintf(os.Stderr, "Error starting API server: %v\n", err) 37 | os.Exit(1) 38 | } 39 | }, 40 | } 41 | 42 | func init() { 43 | // Add flags 44 | serverCmd.Flags().StringVar(&serverHost, "host", "localhost", "Host to bind the API server to") 45 | serverCmd.Flags().IntVar(&serverPort, "port", 8081, "Port to bind the API server to") 46 | serverCmd.Flags().BoolVar(&serverDebug, "debug", false, "Enable debug mode") 47 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build test clean lint run 2 | 3 | # Go parameters 4 | GOCMD=go 5 | GOBUILD=$(GOCMD) build 6 | GOCLEAN=$(GOCMD) clean 7 | GOTEST=$(GOCMD) test 8 | GOGET=$(GOCMD) get 9 | GOMOD=$(GOCMD) mod 10 | BINARY_NAME=movery 11 | BINARY_UNIX=$(BINARY_NAME)_unix 12 | 13 | # Build parameters 14 | BUILD_DIR=go/bin 15 | MAIN_PATH=./go/cmd/movery 16 | 17 | all: test build 18 | 19 | build: 20 | cd go && $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -v $(MAIN_PATH) 21 | 22 | test: 23 | cd go && $(GOTEST) -v ./... 24 | 25 | clean: 26 | cd go && $(GOCLEAN) 27 | rm -f $(BUILD_DIR)/* 28 | 29 | run: 30 | cd go && $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) -v $(MAIN_PATH) 31 | ./$(BUILD_DIR)/$(BINARY_NAME) 32 | 33 | lint: 34 | cd go && golangci-lint run 35 | 36 | deps: 37 | cd go && $(GOMOD) download 38 | 39 | # Cross compilation 40 | build-linux: 41 | cd go && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_UNIX) -v $(MAIN_PATH) 42 | 43 | build-windows: 44 | cd go && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME).exe -v $(MAIN_PATH) 45 | 46 | # Help target 47 | help: 48 | @echo "Available targets:" 49 | @echo " build - Build the project" 50 | @echo " test - Run tests" 51 | @echo " clean - Clean build files" 52 | @echo " run - Build and run the project" 53 | @echo " lint - Run linter" 54 | @echo " deps - Download dependencies" 55 | @echo " build-linux - Build for Linux" 56 | @echo " build-windows- Build for Windows" -------------------------------------------------------------------------------- /go/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/re-movery/re-movery 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/gin-gonic/gin v1.8.1 7 | github.com/spf13/cobra v1.5.0 8 | github.com/stretchr/testify v1.8.0 9 | go.uber.org/zap v1.23.0 10 | ) 11 | 12 | require ( 13 | github.com/davecgh/go-spew v1.1.1 // indirect 14 | github.com/gin-contrib/sse v0.1.0 // indirect 15 | github.com/go-playground/locales v0.14.0 // indirect 16 | github.com/go-playground/universal-translator v0.18.0 // indirect 17 | github.com/go-playground/validator/v10 v10.11.0 // indirect 18 | github.com/goccy/go-json v0.9.10 // indirect 19 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 20 | github.com/json-iterator/go v1.1.12 // indirect 21 | github.com/leodido/go-urn v1.2.1 // indirect 22 | github.com/mattn/go-isatty v0.0.14 // indirect 23 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 24 | github.com/modern-go/reflect2 v1.0.2 // indirect 25 | github.com/pelletier/go-toml/v2 v2.0.2 // indirect 26 | github.com/pmezard/go-difflib v1.0.0 // indirect 27 | github.com/spf13/pflag v1.0.5 // indirect 28 | github.com/ugorji/go/codec v1.2.7 // indirect 29 | go.uber.org/atomic v1.9.0 // indirect 30 | go.uber.org/multierr v1.8.0 // indirect 31 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect 32 | golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect 33 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect 34 | golang.org/x/text v0.3.7 // indirect 35 | google.golang.org/protobuf v1.28.0 // indirect 36 | gopkg.in/yaml.v2 v2.4.0 // indirect 37 | gopkg.in/yaml.v3 v3.0.1 // indirect 38 | ) -------------------------------------------------------------------------------- /go/internal/utils/logging.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "sync" 7 | 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | var ( 12 | logger *logrus.Logger 13 | once sync.Once 14 | ) 15 | 16 | // GetLogger returns the singleton logger instance 17 | func GetLogger() *logrus.Logger { 18 | once.Do(func() { 19 | logger = logrus.New() 20 | logger.SetFormatter(&logrus.TextFormatter{ 21 | FullTimestamp: true, 22 | }) 23 | logger.SetOutput(os.Stdout) 24 | logger.SetLevel(logrus.InfoLevel) 25 | }) 26 | return logger 27 | } 28 | 29 | // FileLogger represents a logger that writes to a file 30 | type FileLogger struct { 31 | *logrus.Logger 32 | file *os.File 33 | } 34 | 35 | // NewFileLogger creates a new file logger 36 | func NewFileLogger(filename string) (*FileLogger, error) { 37 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | logger := logrus.New() 43 | logger.SetFormatter(&logrus.JSONFormatter{}) 44 | logger.SetOutput(io.MultiWriter(file, os.Stdout)) 45 | 46 | return &FileLogger{ 47 | Logger: logger, 48 | file: file, 49 | }, nil 50 | } 51 | 52 | // Close closes the log file 53 | func (fl *FileLogger) Close() error { 54 | if fl.file != nil { 55 | return fl.file.Close() 56 | } 57 | return nil 58 | } 59 | 60 | // SetVerbosity sets the logging level based on verbosity 61 | func SetVerbosity(verbose bool) { 62 | if verbose { 63 | GetLogger().SetLevel(logrus.DebugLevel) 64 | } else { 65 | GetLogger().SetLevel(logrus.InfoLevel) 66 | } 67 | } -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "processing": { 3 | "num_processes": 4, 4 | "max_memory_usage": 8589934592, 5 | "chunk_size": 1048576, 6 | "enable_cache": true, 7 | "cache_dir": ".cache", 8 | "cache_max_size": 1073741824, 9 | "supported_languages": [ 10 | "c", 11 | "cpp", 12 | "java", 13 | "python", 14 | "go", 15 | "javascript" 16 | ] 17 | }, 18 | "detector": { 19 | "min_similarity": 0.8, 20 | "max_edit_distance": 10, 21 | "context_lines": 3, 22 | "max_ast_depth": 50, 23 | "max_cfg_nodes": 1000, 24 | "enable_semantic_match": true, 25 | "enable_syntax_match": true, 26 | "enable_token_match": true, 27 | "report_format": "html", 28 | "report_dir": "reports", 29 | "exclude_patterns": [ 30 | "**/test/*", 31 | "**/tests/*", 32 | "**/vendor/*", 33 | "**/node_modules/*" 34 | ] 35 | }, 36 | "logging": { 37 | "log_level": "INFO", 38 | "log_file": "movery.log", 39 | "log_format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", 40 | "enable_profiling": false, 41 | "profile_output": "profile.stats", 42 | "show_progress": true, 43 | "progress_interval": 1 44 | }, 45 | "security": { 46 | "max_file_size": 104857600, 47 | "allowed_schemes": [ 48 | "file", 49 | "http", 50 | "https" 51 | ], 52 | "enable_sandbox": true, 53 | "sandbox_timeout": 60, 54 | "require_auth": false, 55 | "rate_limit": 100, 56 | "rate_limit_period": 60 57 | } 58 | } -------------------------------------------------------------------------------- /movery/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "processing": { 3 | "num_processes": 4, 4 | "max_memory_usage": 8589934592, 5 | "chunk_size": 1048576, 6 | "enable_cache": true, 7 | "cache_dir": ".cache", 8 | "cache_max_size": 1073741824, 9 | "supported_languages": [ 10 | "c", 11 | "cpp", 12 | "java", 13 | "python", 14 | "go", 15 | "javascript" 16 | ] 17 | }, 18 | "detector": { 19 | "min_similarity": 0.8, 20 | "max_edit_distance": 10, 21 | "context_lines": 3, 22 | "max_ast_depth": 50, 23 | "max_cfg_nodes": 1000, 24 | "enable_semantic_match": true, 25 | "enable_syntax_match": true, 26 | "enable_token_match": true, 27 | "report_format": "html", 28 | "report_dir": "reports", 29 | "exclude_patterns": [ 30 | "**/test/*", 31 | "**/tests/*", 32 | "**/vendor/*", 33 | "**/node_modules/*" 34 | ] 35 | }, 36 | "logging": { 37 | "log_level": "INFO", 38 | "log_file": "movery.log", 39 | "log_format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", 40 | "enable_profiling": false, 41 | "profile_output": "profile.stats", 42 | "show_progress": true, 43 | "progress_interval": 1 44 | }, 45 | "security": { 46 | "max_file_size": 104857600, 47 | "allowed_schemes": [ 48 | "file", 49 | "http", 50 | "https" 51 | ], 52 | "enable_sandbox": true, 53 | "sandbox_timeout": 60, 54 | "require_auth": false, 55 | "rate_limit": 100, 56 | "rate_limit_period": 60 57 | } 58 | } -------------------------------------------------------------------------------- /movery/config/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "processing": { 3 | "num_processes": 4, 4 | "max_memory_usage": 8589934592, 5 | "chunk_size": 1048576, 6 | "enable_cache": true, 7 | "cache_dir": ".cache", 8 | "cache_max_size": 1073741824, 9 | "supported_languages": [ 10 | "c", 11 | "cpp", 12 | "java", 13 | "python", 14 | "go", 15 | "javascript" 16 | ] 17 | }, 18 | "detector": { 19 | "min_similarity": 0.8, 20 | "max_edit_distance": 10, 21 | "context_lines": 3, 22 | "max_ast_depth": 50, 23 | "max_cfg_nodes": 1000, 24 | "enable_semantic_match": true, 25 | "enable_syntax_match": true, 26 | "enable_token_match": true, 27 | "report_format": "html", 28 | "report_dir": "reports", 29 | "exclude_patterns": [ 30 | "**/test/*", 31 | "**/tests/*", 32 | "**/vendor/*", 33 | "**/node_modules/*" 34 | ] 35 | }, 36 | "logging": { 37 | "log_level": "INFO", 38 | "log_file": "movery.log", 39 | "log_format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", 40 | "enable_profiling": false, 41 | "profile_output": "profile.stats", 42 | "show_progress": true, 43 | "progress_interval": 1 44 | }, 45 | "security": { 46 | "max_file_size": 104857600, 47 | "allowed_schemes": [ 48 | "file", 49 | "http", 50 | "https" 51 | ], 52 | "enable_sandbox": true, 53 | "sandbox_timeout": 60, 54 | "require_auth": false, 55 | "rate_limit": 100, 56 | "rate_limit_period": 60 57 | } 58 | } -------------------------------------------------------------------------------- /go/internal/utils/parallel.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // Job represents a unit of work 8 | type Job interface { 9 | Execute() error 10 | } 11 | 12 | // WorkerPool manages a pool of workers for parallel processing 13 | type WorkerPool struct { 14 | numWorkers int 15 | jobs chan Job 16 | results chan error 17 | wg sync.WaitGroup 18 | stopChan chan struct{} 19 | } 20 | 21 | // NewWorkerPool creates a new worker pool 22 | func NewWorkerPool(numWorkers int, queueSize int) *WorkerPool { 23 | return &WorkerPool{ 24 | numWorkers: numWorkers, 25 | jobs: make(chan Job, queueSize), 26 | results: make(chan error, queueSize), 27 | stopChan: make(chan struct{}), 28 | } 29 | } 30 | 31 | // Start starts the worker pool 32 | func (wp *WorkerPool) Start() { 33 | for i := 0; i < wp.numWorkers; i++ { 34 | wp.wg.Add(1) 35 | go wp.worker() 36 | } 37 | } 38 | 39 | // worker processes jobs from the job queue 40 | func (wp *WorkerPool) worker() { 41 | defer wp.wg.Done() 42 | 43 | for { 44 | select { 45 | case job := <-wp.jobs: 46 | if job == nil { 47 | return 48 | } 49 | err := job.Execute() 50 | wp.results <- err 51 | case <-wp.stopChan: 52 | return 53 | } 54 | } 55 | } 56 | 57 | // Submit submits a job to the worker pool 58 | func (wp *WorkerPool) Submit(job Job) { 59 | wp.jobs <- job 60 | } 61 | 62 | // Stop stops the worker pool 63 | func (wp *WorkerPool) Stop() { 64 | close(wp.stopChan) 65 | wp.wg.Wait() 66 | close(wp.jobs) 67 | close(wp.results) 68 | } 69 | 70 | // Results returns the results channel 71 | func (wp *WorkerPool) Results() <-chan error { 72 | return wp.results 73 | } -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Setup script for Re-Movery 3 | """ 4 | from setuptools import setup, find_packages 5 | 6 | with open("README.md", "r", encoding="utf-8") as f: 7 | long_description = f.read() 8 | 9 | setup( 10 | name="movery", 11 | version="0.1.0", 12 | author="heyangxu", 13 | author_email="", 14 | description="A tool for discovering modified vulnerable code clones", 15 | long_description=long_description, 16 | long_description_content_type="text/markdown", 17 | url="https://github.com/heyangxu/Re-movery", 18 | packages=find_packages(), 19 | classifiers=[ 20 | "Development Status :: 4 - Beta", 21 | "Intended Audience :: Developers", 22 | "Topic :: Security", 23 | "Topic :: Software Development :: Quality Assurance", 24 | "License :: OSI Approved :: MIT License", 25 | "Programming Language :: Python :: 3", 26 | "Programming Language :: Python :: 3.7", 27 | "Programming Language :: Python :: 3.8", 28 | "Programming Language :: Python :: 3.9", 29 | "Programming Language :: Python :: 3.10", 30 | "Operating System :: OS Independent", 31 | ], 32 | python_requires=">=3.7", 33 | install_requires=[ 34 | "pytest>=7.3.1", 35 | "coverage>=7.2.7", 36 | "jinja2>=3.0.0", 37 | "plotly>=5.0.0", 38 | "pandas>=1.3.0", 39 | "psutil>=5.8.0", 40 | "tqdm>=4.61.0", 41 | "colorama>=0.4.4", 42 | "requests>=2.26.0", 43 | "beautifulsoup4>=4.9.3", 44 | "lxml>=4.6.3", 45 | "pygments>=2.9.0", 46 | "typing-extensions>=3.10.0", 47 | "dataclasses>=0.8;python_version<'3.7'", 48 | ], 49 | entry_points={ 50 | "console_scripts": [ 51 | "movery=movery.main:main", 52 | ], 53 | }, 54 | package_data={ 55 | "movery": [ 56 | "templates/*.html", 57 | "config/*.json", 58 | ], 59 | }, 60 | include_package_data=True, 61 | zip_safe=False, 62 | ) -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | name: Build and Test 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v4 19 | with: 20 | go-version: '1.21' 21 | cache: true 22 | 23 | - name: Install dependencies 24 | run: cd go && go mod download 25 | 26 | - name: Run golangci-lint 27 | uses: golangci/golangci-lint-action@v3 28 | with: 29 | version: latest 30 | working-directory: go 31 | args: --timeout=5m 32 | 33 | - name: Run tests 34 | run: cd go && go test -v ./... -coverprofile=coverage.txt -covermode=atomic 35 | 36 | - name: Upload coverage to Codecov 37 | uses: codecov/codecov-action@v3 38 | with: 39 | file: ./go/coverage.txt 40 | flags: unittests 41 | 42 | - name: Build 43 | run: cd go && go build -v ./cmd/movery 44 | 45 | release: 46 | name: Create Release 47 | needs: build 48 | runs-on: ubuntu-latest 49 | if: startsWith(github.ref, 'refs/tags/') 50 | 51 | steps: 52 | - uses: actions/checkout@v3 53 | 54 | - name: Set up Go 55 | uses: actions/setup-go@v4 56 | with: 57 | go-version: '1.21' 58 | 59 | - name: Build for multiple platforms 60 | run: | 61 | cd go 62 | GOOS=linux GOARCH=amd64 go build -o movery-linux-amd64 ./cmd/movery 63 | GOOS=windows GOARCH=amd64 go build -o movery-windows-amd64.exe ./cmd/movery 64 | GOOS=darwin GOARCH=amd64 go build -o movery-darwin-amd64 ./cmd/movery 65 | 66 | - name: Create Release 67 | uses: softprops/action-gh-release@v1 68 | with: 69 | files: | 70 | go/movery-linux-amd64 71 | go/movery-windows-amd64.exe 72 | go/movery-darwin-amd64 73 | env: 74 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # 贡献指南 2 | 3 | 感谢您对Re-movery项目的关注！我们欢迎任何形式的贡献，包括但不限于： 4 | 5 | - 报告问题 6 | - 提交功能建议 7 | - 改进文档 8 | - 提交代码修复 9 | - 添加新功能 10 | 11 | ## 开发环境设置 12 | 13 | 1. 安装Go 1.21或更高版本 14 | 2. 克隆仓库： 15 | ```bash 16 | git clone https://github.com/heyangxu/Re-movery.git 17 | cd Re-movery 18 | ``` 19 | 3. 安装依赖： 20 | ```bash 21 | cd go 22 | go mod download 23 | ``` 24 | 4. 安装开发工具： 25 | ```bash 26 | # 安装golangci-lint 27 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 28 | ``` 29 | 30 | ## 开发流程 31 | 32 | 1. 创建新分支： 33 | ```bash 34 | git checkout -b feature/your-feature-name 35 | ``` 36 | 37 | 2. 进行开发，确保： 38 | - 遵循Go代码规范 39 | - 添加适当的测试 40 | - 更新相关文档 41 | 42 | 3. 运行测试： 43 | ```bash 44 | make test 45 | ``` 46 | 47 | 4. 运行代码检查： 48 | ```bash 49 | make lint 50 | ``` 51 | 52 | 5. 提交代码： 53 | ```bash 54 | git add . 55 | git commit -m "feat: Add your feature description" 56 | ``` 57 | 58 | 6. 推送到GitHub： 59 | ```bash 60 | git push origin feature/your-feature-name 61 | ``` 62 | 63 | 7. 创建Pull Request 64 | 65 | ## 提交规范 66 | 67 | 我们使用[Conventional Commits](https://www.conventionalcommits.org/)规范，提交信息格式如下： 68 | 69 | ``` 70 | (): 71 | 72 | [optional body] 73 | 74 | [optional footer] 75 | ``` 76 | 77 | 类型（type）包括： 78 | - feat: 新功能 79 | - fix: 修复 80 | - docs: 文档更新 81 | - style: 代码格式（不影响代码运行的变动） 82 | - refactor: 重构 83 | - perf: 性能优化 84 | - test: 测试 85 | - chore: 构建过程或辅助工具的变动 86 | 87 | ## 代码规范 88 | 89 | - 遵循[Go代码规范](https://golang.org/doc/effective_go) 90 | - 使用`gofmt`格式化代码 91 | - 添加适当的注释 92 | - 保持代码简洁明了 93 | - 使用有意义的变量和函数名 94 | 95 | ## 测试规范 96 | 97 | - 为新功能添加单元测试 98 | - 确保测试覆盖率不降低 99 | - 测试应该简单明了 100 | - 避免测试之间的依赖 101 | 102 | ## 文档规范 103 | 104 | - 保持README.md的更新 105 | - 为新功能添加文档 106 | - 更新API文档 107 | - 添加示例代码 108 | 109 | ## 问题反馈 110 | 111 | 如果您发现了问题或有新的想法，请： 112 | 113 | 1. 检查是否已存在相关的Issue 114 | 2. 如果没有，创建新的Issue 115 | 3. 清晰描述问题或建议 116 | 4. 提供复现步骤（如果适用） 117 | 5. 提供相关的日志或截图（如果适用） 118 | 119 | ## 行为准则 120 | 121 | 请参阅我们的[行为准则](CODE_OF_CONDUCT.md)。 122 | 123 | ## 许可证 124 | 125 | 通过提交代码，您同意您的代码遵循项目的[MIT许可证](LICENSE)。 -------------------------------------------------------------------------------- /movery/config/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration module for Movery 3 | """ 4 | import json 5 | import os 6 | from typing import Dict, Any, List 7 | from dataclasses import dataclass 8 | 9 | @dataclass 10 | class ProcessingConfig: 11 | num_processes: int 12 | max_memory_usage: int 13 | chunk_size: int 14 | enable_cache: bool 15 | cache_dir: str 16 | cache_max_size: int 17 | supported_languages: List[str] 18 | 19 | @dataclass 20 | class DetectorConfig: 21 | min_similarity: float 22 | max_edit_distance: int 23 | context_lines: int 24 | max_ast_depth: int 25 | max_cfg_nodes: int 26 | enable_semantic_match: bool 27 | enable_syntax_match: bool 28 | enable_token_match: bool 29 | report_format: str 30 | report_dir: str 31 | exclude_patterns: List[str] 32 | 33 | @dataclass 34 | class LoggingConfig: 35 | log_level: str 36 | log_file: str 37 | log_format: str 38 | enable_profiling: bool 39 | profile_output: str 40 | show_progress: bool 41 | progress_interval: int 42 | 43 | @dataclass 44 | class SecurityConfig: 45 | max_file_size: int 46 | allowed_schemes: List[str] 47 | enable_sandbox: bool 48 | sandbox_timeout: int 49 | require_auth: bool 50 | rate_limit: int 51 | rate_limit_period: int 52 | 53 | @dataclass 54 | class Config: 55 | processing: ProcessingConfig 56 | detector: DetectorConfig 57 | logging: LoggingConfig 58 | security: SecurityConfig 59 | 60 | def load_config(config_path: str = None) -> Config: 61 | """ 62 | Load configuration from JSON file 63 | 64 | Args: 65 | config_path: Path to config file. If None, uses default config.json 66 | 67 | Returns: 68 | Configuration object 69 | """ 70 | if config_path is None: 71 | config_path = os.path.join(os.path.dirname(__file__), "config.json") 72 | 73 | with open(config_path, "r", encoding="utf-8") as f: 74 | data = json.load(f) 75 | 76 | return Config( 77 | processing=ProcessingConfig(**data["processing"]), 78 | detector=DetectorConfig(**data["detector"]), 79 | logging=LoggingConfig(**data["logging"]), 80 | security=SecurityConfig(**data["security"]) 81 | ) 82 | 83 | # Load default configuration 84 | config = load_config() -------------------------------------------------------------------------------- /go/internal/web/static/css/style.css: -------------------------------------------------------------------------------- 1 | /* Re-movery 样式文件 */ 2 | 3 | body { 4 | font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; 5 | background-color: #f8f9fa; 6 | } 7 | 8 | .navbar-brand { 9 | font-weight: bold; 10 | color: #0d6efd; 11 | } 12 | 13 | .card { 14 | border-radius: 10px; 15 | box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1); 16 | margin-bottom: 20px; 17 | } 18 | 19 | .card-header { 20 | font-weight: bold; 21 | background-color: #f8f9fa; 22 | } 23 | 24 | .severity-high { 25 | color: #dc3545; 26 | } 27 | 28 | .severity-medium { 29 | color: #fd7e14; 30 | } 31 | 32 | .severity-low { 33 | color: #0dcaf0; 34 | } 35 | 36 | .chart-container { 37 | height: 300px; 38 | } 39 | 40 | .nav-pills .nav-link.active { 41 | background-color: #0d6efd; 42 | } 43 | 44 | .nav-pills .nav-link { 45 | color: #495057; 46 | } 47 | 48 | .file-item { 49 | cursor: pointer; 50 | } 51 | 52 | .file-item:hover { 53 | background-color: #f8f9fa; 54 | } 55 | 56 | .code-block { 57 | background-color: #f8f9fa; 58 | border-radius: 5px; 59 | padding: 10px; 60 | font-family: monospace; 61 | white-space: pre-wrap; 62 | margin-top: 10px; 63 | } 64 | 65 | .footer { 66 | margin-top: 50px; 67 | padding: 20px 0; 68 | background-color: #f8f9fa; 69 | text-align: center; 70 | color: #6c757d; 71 | } 72 | 73 | /* 按钮样式 */ 74 | .btn-primary { 75 | background-color: #0d6efd; 76 | border-color: #0d6efd; 77 | } 78 | 79 | .btn-primary:hover { 80 | background-color: #0b5ed7; 81 | border-color: #0a58ca; 82 | } 83 | 84 | /* 表单样式 */ 85 | .form-control:focus { 86 | border-color: #0d6efd; 87 | box-shadow: 0 0 0 0.25rem rgba(13, 110, 253, 0.25); 88 | } 89 | 90 | /* 表格样式 */ 91 | .table { 92 | border-collapse: collapse; 93 | width: 100%; 94 | } 95 | 96 | .table th { 97 | background-color: #f8f9fa; 98 | font-weight: bold; 99 | } 100 | 101 | .table-striped tbody tr:nth-of-type(odd) { 102 | background-color: rgba(0, 0, 0, 0.05); 103 | } 104 | 105 | /* 徽章样式 */ 106 | .badge { 107 | font-weight: normal; 108 | padding: 0.35em 0.65em; 109 | } 110 | 111 | /* 响应式调整 */ 112 | @media (max-width: 768px) { 113 | .chart-container { 114 | height: 200px; 115 | } 116 | } -------------------------------------------------------------------------------- /go/internal/core/models.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Signature represents a vulnerability signature 8 | type Signature struct { 9 | ID string `json:"id"` 10 | Name string `json:"name"` 11 | Severity string `json:"severity"` 12 | Description string `json:"description"` 13 | CodePatterns []string `json:"codePatterns"` 14 | References []string `json:"references"` 15 | } 16 | 17 | // Match represents a vulnerability match 18 | type Match struct { 19 | Signature Signature `json:"signature"` 20 | FilePath string `json:"filePath"` 21 | LineNumber int `json:"lineNumber"` 22 | MatchedCode string `json:"matchedCode"` 23 | Confidence float64 `json:"confidence"` 24 | } 25 | 26 | // Summary represents a summary of scan results 27 | type Summary struct { 28 | TotalFiles int `json:"totalFiles"` 29 | High int `json:"high"` 30 | Medium int `json:"medium"` 31 | Low int `json:"low"` 32 | Vulnerabilities map[string]int `json:"vulnerabilities"` 33 | } 34 | 35 | // ReportData represents data for a report 36 | type ReportData struct { 37 | Title string `json:"title"` 38 | Timestamp string `json:"timestamp"` 39 | Results map[string][]Match `json:"results"` 40 | Summary Summary `json:"summary"` 41 | } 42 | 43 | // Reporter is an interface for report generators 44 | type Reporter interface { 45 | GenerateReport(data ReportData, outputPath string) error 46 | } 47 | 48 | // Detector is an interface for vulnerability detectors 49 | type Detector interface { 50 | Name() string 51 | SupportedLanguages() []string 52 | DetectFile(filePath string) ([]Match, error) 53 | DetectCode(code string, filePath string) ([]Match, error) 54 | } 55 | 56 | // GenerateSummary generates a summary from scan results 57 | func GenerateSummary(results map[string][]Match) Summary { 58 | summary := Summary{ 59 | TotalFiles: len(results), 60 | Vulnerabilities: make(map[string]int), 61 | } 62 | 63 | for _, matches := range results { 64 | for _, match := range matches { 65 | switch match.Signature.Severity { 66 | case "high": 67 | summary.High++ 68 | case "medium": 69 | summary.Medium++ 70 | case "low": 71 | summary.Low++ 72 | } 73 | 74 | // Count vulnerabilities by name 75 | summary.Vulnerabilities[match.Signature.Name]++ 76 | } 77 | } 78 | 79 | return summary 80 | } -------------------------------------------------------------------------------- /movery/analyzers/code_analyzer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code analysis utilities for Movery 3 | """ 4 | from typing import Dict, List, Optional 5 | import os 6 | import ast 7 | import logging 8 | 9 | from movery.utils.logging import get_logger 10 | from movery.config.config import config 11 | from .language import LanguageAnalyzer, PythonAnalyzer, JavaAnalyzer, CppAnalyzer, GoAnalyzer 12 | 13 | logger = get_logger(__name__) 14 | 15 | class CodeAnalyzer: 16 | """Code analyzer that supports multiple programming languages""" 17 | 18 | def __init__(self): 19 | self.analyzers = { 20 | ".py": PythonAnalyzer(), 21 | ".java": JavaAnalyzer(), 22 | ".cpp": CppAnalyzer(), 23 | ".hpp": CppAnalyzer(), 24 | ".cc": CppAnalyzer(), 25 | ".hh": CppAnalyzer(), 26 | ".go": GoAnalyzer() 27 | } 28 | 29 | def analyze_file(self, filename: str) -> Dict: 30 | """Analyze a source code file""" 31 | ext = os.path.splitext(filename)[1].lower() 32 | 33 | if ext not in self.analyzers: 34 | logger.warning(f"Unsupported file type: {ext}") 35 | return { 36 | "complexity": 0, 37 | "functions": [], 38 | "classes": [], 39 | "imports": [], 40 | "variables": [] 41 | } 42 | 43 | analyzer = self.analyzers[ext] 44 | try: 45 | ast_node = analyzer.parse_file(filename) 46 | 47 | return { 48 | "complexity": self._calculate_complexity(ast_node), 49 | "functions": analyzer.get_functions(ast_node), 50 | "classes": analyzer.get_classes(ast_node), 51 | "imports": analyzer.get_imports(ast_node), 52 | "variables": analyzer.get_variables(ast_node) 53 | } 54 | 55 | except Exception as e: 56 | logger.error(f"Error analyzing file {filename}: {str(e)}") 57 | return { 58 | "complexity": 0, 59 | "functions": [], 60 | "classes": [], 61 | "imports": [], 62 | "variables": [] 63 | } 64 | 65 | def _calculate_complexity(self, ast_node: any) -> int: 66 | """Calculate code complexity""" 67 | # 简单实现 - 仅计算函数和类的数量 68 | if isinstance(ast_node, ast.AST): 69 | functions = sum(1 for node in ast.walk(ast_node) 70 | if isinstance(node, ast.FunctionDef)) 71 | classes = sum(1 for node in ast.walk(ast_node) 72 | if isinstance(node, ast.ClassDef)) 73 | return functions + classes 74 | return 0 -------------------------------------------------------------------------------- /go/internal/analyzers/language.go: -------------------------------------------------------------------------------- 1 | package analyzers 2 | 3 | import ( 4 | "go/ast" 5 | "go/parser" 6 | "go/token" 7 | "path/filepath" 8 | ) 9 | 10 | // LanguageAnalyzer defines the interface for language analyzers 11 | type LanguageAnalyzer interface { 12 | ParseFile(filename string) (ast.Node, error) 13 | ExtractFunctions(node ast.Node) []ast.Node 14 | ExtractClasses(node ast.Node) []ast.Node 15 | ExtractImports(node ast.Node) []string 16 | ExtractVariables(node ast.Node) []ast.Node 17 | } 18 | 19 | // GoAnalyzer implements LanguageAnalyzer for Go language 20 | type GoAnalyzer struct { 21 | fset *token.FileSet 22 | } 23 | 24 | // NewGoAnalyzer creates a new Go language analyzer 25 | func NewGoAnalyzer() *GoAnalyzer { 26 | return &GoAnalyzer{ 27 | fset: token.NewFileSet(), 28 | } 29 | } 30 | 31 | // ParseFile parses a Go source file 32 | func (ga *GoAnalyzer) ParseFile(filename string) (ast.Node, error) { 33 | return parser.ParseFile(ga.fset, filename, nil, parser.AllErrors) 34 | } 35 | 36 | // ExtractFunctions extracts function declarations from an AST 37 | func (ga *GoAnalyzer) ExtractFunctions(node ast.Node) []ast.Node { 38 | var functions []ast.Node 39 | ast.Inspect(node, func(n ast.Node) bool { 40 | if fn, ok := n.(*ast.FuncDecl); ok { 41 | functions = append(functions, fn) 42 | } 43 | return true 44 | }) 45 | return functions 46 | } 47 | 48 | // ExtractClasses extracts type declarations from an AST 49 | func (ga *GoAnalyzer) ExtractClasses(node ast.Node) []ast.Node { 50 | var types []ast.Node 51 | ast.Inspect(node, func(n ast.Node) bool { 52 | if t, ok := n.(*ast.TypeSpec); ok { 53 | types = append(types, t) 54 | } 55 | return true 56 | }) 57 | return types 58 | } 59 | 60 | // ExtractImports extracts import declarations from an AST 61 | func (ga *GoAnalyzer) ExtractImports(node ast.Node) []string { 62 | var imports []string 63 | ast.Inspect(node, func(n ast.Node) bool { 64 | if imp, ok := n.(*ast.ImportSpec); ok { 65 | imports = append(imports, imp.Path.Value) 66 | } 67 | return true 68 | }) 69 | return imports 70 | } 71 | 72 | // ExtractVariables extracts variable declarations from an AST 73 | func (ga *GoAnalyzer) ExtractVariables(node ast.Node) []ast.Node { 74 | var variables []ast.Node 75 | ast.Inspect(node, func(n ast.Node) bool { 76 | if v, ok := n.(*ast.ValueSpec); ok { 77 | variables = append(variables, v) 78 | } 79 | return true 80 | }) 81 | return variables 82 | } 83 | 84 | // GetFileLanguage determines the programming language of a file 85 | func GetFileLanguage(filename string) string { 86 | ext := filepath.Ext(filename) 87 | switch ext { 88 | case ".go": 89 | return "go" 90 | case ".java": 91 | return "java" 92 | case ".py": 93 | return "python" 94 | case ".js": 95 | return "javascript" 96 | case ".ts": 97 | return "typescript" 98 | default: 99 | return "unknown" 100 | } 101 | } -------------------------------------------------------------------------------- /go/README.md: -------------------------------------------------------------------------------- 1 | # Re-movery (Go版本) 2 | 3 | Re-movery是一个强大的安全漏洞扫描工具，用于检测代码中的潜在安全问题。Go版本提供了高性能的扫描能力和多种接口选项。 4 | 5 | ## 功能特点 6 | 7 | - 支持多种编程语言（目前支持Python和JavaScript） 8 | - 提供命令行、Web界面和API接口 9 | - 生成HTML、JSON和XML格式的报告 10 | - 支持并行扫描和增量扫描 11 | - 与CI/CD工具集成（GitHub Actions、GitLab CI） 12 | - VS Code扩展支持 13 | 14 | ## 安装 15 | 16 | ### 从源码安装 17 | 18 | ```bash 19 | git clone https://github.com/re-movery/re-movery.git 20 | cd re-movery/go 21 | go install ./cmd/movery 22 | ``` 23 | 24 | ### 使用Go工具安装 25 | 26 | ```bash 27 | go install github.com/re-movery/re-movery/cmd/movery@latest 28 | ``` 29 | 30 | ## 使用方法 31 | 32 | ### 命令行扫描 33 | 34 | ```bash 35 | # 扫描单个文件 36 | movery scan --file path/to/file.py 37 | 38 | # 扫描目录 39 | movery scan --dir path/to/directory 40 | 41 | # 排除特定文件或目录 42 | movery scan --dir path/to/directory --exclude "node_modules,*.min.js" 43 | 44 | # 生成HTML报告 45 | movery scan --dir path/to/directory --output report.html 46 | 47 | # 启用并行处理 48 | movery scan --dir path/to/directory --parallel 49 | 50 | # 启用增量扫描 51 | movery scan --dir path/to/directory --incremental 52 | ``` 53 | 54 | ### 启动Web界面 55 | 56 | ```bash 57 | # 默认配置（localhost:8080） 58 | movery web 59 | 60 | # 自定义主机和端口 61 | movery web --host 0.0.0.0 --port 8080 62 | 63 | # 启用调试模式 64 | movery web --debug 65 | ``` 66 | 67 | ### 启动API服务器 68 | 69 | ```bash 70 | # 默认配置（localhost:8081） 71 | movery server 72 | 73 | # 自定义主机和端口 74 | movery server --host 0.0.0.0 --port 8081 75 | 76 | # 启用调试模式 77 | movery server --debug 78 | ``` 79 | 80 | ### 生成集成文件 81 | 82 | ```bash 83 | # 生成GitHub Actions工作流文件 84 | movery generate github-action 85 | 86 | # 生成GitLab CI配置文件 87 | movery generate gitlab-ci 88 | 89 | # 生成VS Code扩展配置文件 90 | movery generate vscode-extension 91 | ``` 92 | 93 | ## API文档 94 | 95 | ### 扫描代码 96 | 97 | ``` 98 | POST /api/scan/code 99 | Content-Type: application/json 100 | 101 | { 102 | "code": "代码内容", 103 | "language": "python", 104 | "fileName": "example.py" 105 | } 106 | ``` 107 | 108 | ### 扫描文件 109 | 110 | ``` 111 | POST /api/scan/file 112 | Content-Type: multipart/form-data 113 | 114 | file: [文件内容] 115 | ``` 116 | 117 | ### 扫描目录 118 | 119 | ``` 120 | POST /api/scan/directory 121 | Content-Type: application/json 122 | 123 | { 124 | "directory": "/path/to/directory", 125 | "excludePatterns": ["node_modules", "*.min.js"], 126 | "parallel": true, 127 | "incremental": false 128 | } 129 | ``` 130 | 131 | ### 获取支持的语言 132 | 133 | ``` 134 | GET /api/languages 135 | ``` 136 | 137 | ## 配置 138 | 139 | Re-movery可以通过命令行参数或配置文件进行配置。配置文件支持YAML、JSON和TOML格式。 140 | 141 | ```yaml 142 | # re-movery.yaml 143 | scanner: 144 | parallel: true 145 | incremental: true 146 | confidenceThreshold: 0.7 147 | 148 | web: 149 | host: localhost 150 | port: 8080 151 | debug: false 152 | 153 | server: 154 | host: localhost 155 | port: 8081 156 | debug: false 157 | ``` 158 | 159 | ## 开发 160 | 161 | ### 构建 162 | 163 | ```bash 164 | cd go 165 | go build -o movery ./cmd/movery 166 | ``` 167 | 168 | ### 测试 169 | 170 | ```bash 171 | go test ./... 172 | ``` 173 | 174 | ### 贡献 175 | 176 | 欢迎提交Pull Request和Issue。请确保您的代码符合Go的代码规范，并通过所有测试。 177 | 178 | ## 许可证 179 | 180 | MIT -------------------------------------------------------------------------------- /movery/tests/unit/test_detector.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 5 | 6 | from movery.detectors.vulnerability import VulnerabilityDetector 7 | 8 | class TestVulnerabilityDetector(unittest.TestCase): 9 | def setUp(self): 10 | self.detector = VulnerabilityDetector() 11 | self.test_data_dir = os.path.join(os.path.dirname(__file__), 'test_data') 12 | if not os.path.exists(self.test_data_dir): 13 | os.makedirs(self.test_data_dir) 14 | 15 | def test_load_signatures(self): 16 | """测试加载漏洞签名""" 17 | # 创建测试签名文件 18 | test_sig_file = os.path.join(self.test_data_dir, 'test_signatures.json') 19 | with open(test_sig_file, 'w') as f: 20 | f.write(''' 21 | { 22 | "signatures": [ 23 | { 24 | "id": "CWE-78", 25 | "name": "OS Command Injection", 26 | "severity": "HIGH", 27 | "code_patterns": ["os\\.system\\(.*\\)"] 28 | } 29 | ] 30 | } 31 | ''') 32 | 33 | self.detector.load_signatures(test_sig_file) 34 | self.assertEqual(len(self.detector.signatures), 1) 35 | self.assertEqual(self.detector.signatures[0].id, "CWE-78") 36 | 37 | def test_detect_vulnerability(self): 38 | """测试漏洞检测""" 39 | # 创建测试代码文件 40 | test_code_file = os.path.join(self.test_data_dir, 'test_code.py') 41 | with open(test_code_file, 'w') as f: 42 | f.write(''' 43 | import os 44 | def unsafe_function(cmd): 45 | os.system(cmd) # 不安全的系统命令执行 46 | ''') 47 | 48 | matches = self.detector.detect_file(test_code_file) 49 | self.assertTrue(len(matches) > 0) 50 | self.assertEqual(matches[0].signature.id, "CWE-78") 51 | 52 | def test_false_positive(self): 53 | """测试误报情况""" 54 | # 创建安全的测试代码 55 | test_safe_file = os.path.join(self.test_data_dir, 'test_safe.py') 56 | with open(test_safe_file, 'w') as f: 57 | f.write(''' 58 | def safe_function(): 59 | print("This is safe code") 60 | ''') 61 | 62 | matches = self.detector.detect_file(test_safe_file) 63 | self.assertEqual(len(matches), 0) 64 | 65 | def test_similarity_matching(self): 66 | """测试相似度匹配""" 67 | # 创建相似代码测试文件 68 | test_similar_file = os.path.join(self.test_data_dir, 'test_similar.py') 69 | with open(test_similar_file, 'w') as f: 70 | f.write(''' 71 | import subprocess 72 | def similar_unsafe(command): 73 | subprocess.call(command, shell=True) # 类似的不安全模式 74 | ''') 75 | 76 | matches = self.detector.detect_file(test_similar_file) 77 | self.assertTrue(len(matches) > 0) 78 | self.assertTrue(matches[0].confidence > 0.7) 79 | 80 | def tearDown(self): 81 | """清理测试数据""" 82 | import shutil 83 | if os.path.exists(self.test_data_dir): 84 | shutil.rmtree(self.test_data_dir) 85 | 86 | if __name__ == '__main__': 87 | unittest.main() -------------------------------------------------------------------------------- /go/internal/utils/memory.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "container/list" 5 | "runtime" 6 | "sync" 7 | "time" 8 | 9 | "github.com/shirou/gopsutil/v3/mem" 10 | ) 11 | 12 | // MemoryMonitor monitors system memory usage 13 | type MemoryMonitor struct { 14 | maxMemoryGB float64 15 | interval time.Duration 16 | stopChan chan struct{} 17 | } 18 | 19 | // NewMemoryMonitor creates a new memory monitor 20 | func NewMemoryMonitor(maxMemoryGB float64, interval time.Duration) *MemoryMonitor { 21 | return &MemoryMonitor{ 22 | maxMemoryGB: maxMemoryGB, 23 | interval: interval, 24 | stopChan: make(chan struct{}), 25 | } 26 | } 27 | 28 | // Start starts monitoring memory usage 29 | func (mm *MemoryMonitor) Start() { 30 | go func() { 31 | ticker := time.NewTicker(mm.interval) 32 | defer ticker.Stop() 33 | 34 | for { 35 | select { 36 | case <-ticker.C: 37 | v, err := mem.VirtualMemory() 38 | if err != nil { 39 | GetLogger().Errorf("Failed to get memory stats: %v", err) 40 | continue 41 | } 42 | 43 | usedGB := float64(v.Used) / (1024 * 1024 * 1024) 44 | if usedGB > mm.maxMemoryGB { 45 | GetLogger().Warnf("Memory usage (%.2f GB) exceeds limit (%.2f GB), triggering GC", usedGB, mm.maxMemoryGB) 46 | runtime.GC() 47 | } 48 | case <-mm.stopChan: 49 | return 50 | } 51 | } 52 | }() 53 | } 54 | 55 | // Stop stops the memory monitor 56 | func (mm *MemoryMonitor) Stop() { 57 | close(mm.stopChan) 58 | } 59 | 60 | // LRUCache implements a thread-safe LRU cache 61 | type LRUCache struct { 62 | capacity int 63 | cache map[interface{}]*list.Element 64 | ll *list.List 65 | mutex sync.RWMutex 66 | } 67 | 68 | type entry struct { 69 | key interface{} 70 | value interface{} 71 | } 72 | 73 | // NewLRUCache creates a new LRU cache with the specified capacity 74 | func NewLRUCache(capacity int) *LRUCache { 75 | return &LRUCache{ 76 | capacity: capacity, 77 | cache: make(map[interface{}]*list.Element), 78 | ll: list.New(), 79 | } 80 | } 81 | 82 | // Get retrieves a value from the cache 83 | func (c *LRUCache) Get(key interface{}) (interface{}, bool) { 84 | c.mutex.RLock() 85 | defer c.mutex.RUnlock() 86 | 87 | if elem, ok := c.cache[key]; ok { 88 | c.ll.MoveToFront(elem) 89 | return elem.Value.(*entry).value, true 90 | } 91 | return nil, false 92 | } 93 | 94 | // Put adds a value to the cache 95 | func (c *LRUCache) Put(key, value interface{}) { 96 | c.mutex.Lock() 97 | defer c.mutex.Unlock() 98 | 99 | if elem, ok := c.cache[key]; ok { 100 | c.ll.MoveToFront(elem) 101 | elem.Value.(*entry).value = value 102 | return 103 | } 104 | 105 | if c.ll.Len() >= c.capacity { 106 | oldest := c.ll.Back() 107 | if oldest != nil { 108 | c.ll.Remove(oldest) 109 | delete(c.cache, oldest.Value.(*entry).key) 110 | } 111 | } 112 | 113 | elem := c.ll.PushFront(&entry{key, value}) 114 | c.cache[key] = elem 115 | } -------------------------------------------------------------------------------- /movery/tests/unit/test_analyzer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 5 | 6 | from movery.analyzers.code_analyzer import CodeAnalyzer 7 | 8 | class TestCodeAnalyzer(unittest.TestCase): 9 | def setUp(self): 10 | self.analyzer = CodeAnalyzer() 11 | self.test_data_dir = os.path.join(os.path.dirname(__file__), 'test_data') 12 | if not os.path.exists(self.test_data_dir): 13 | os.makedirs(self.test_data_dir) 14 | 15 | def test_parse_python(self): 16 | """测试Python代码解析""" 17 | test_file = os.path.join(self.test_data_dir, 'test_python.py') 18 | with open(test_file, 'w') as f: 19 | f.write(''' 20 | def example_function(): 21 | x = 1 22 | y = 2 23 | return x + y 24 | ''') 25 | 26 | ast = self.analyzer.parse_file(test_file) 27 | self.assertIsNotNone(ast) 28 | self.assertEqual(ast.type, 'Module') 29 | 30 | def test_analyze_function(self): 31 | """测试函数分析""" 32 | test_file = os.path.join(self.test_data_dir, 'test_function.py') 33 | with open(test_file, 'w') as f: 34 | f.write(''' 35 | def process_data(data): 36 | result = [] 37 | for item in data: 38 | if item > 0: 39 | result.append(item * 2) 40 | return result 41 | ''') 42 | 43 | functions = self.analyzer.analyze_functions(test_file) 44 | self.assertEqual(len(functions), 1) 45 | self.assertEqual(functions[0].name, 'process_data') 46 | self.assertTrue(functions[0].has_loop) 47 | self.assertTrue(functions[0].has_condition) 48 | 49 | def test_data_flow(self): 50 | """测试数据流分析""" 51 | test_file = os.path.join(self.test_data_dir, 'test_dataflow.py') 52 | with open(test_file, 'w') as f: 53 | f.write(''' 54 | def data_flow_example(user_input): 55 | data = user_input.strip() 56 | processed = data.lower() 57 | return processed 58 | ''') 59 | 60 | flows = self.analyzer.analyze_data_flow(test_file) 61 | self.assertTrue(len(flows) > 0) 62 | self.assertIn('user_input', flows[0].sources) 63 | self.assertIn('processed', flows[0].sinks) 64 | 65 | def test_complexity_analysis(self): 66 | """测试复杂度分析""" 67 | test_file = os.path.join(self.test_data_dir, 'test_complexity.py') 68 | with open(test_file, 'w') as f: 69 | f.write(''' 70 | def complex_function(x, y): 71 | if x > 0: 72 | if y > 0: 73 | return x + y 74 | else: 75 | return x - y 76 | else: 77 | if y < 0: 78 | return -x - y 79 | else: 80 | return -x + y 81 | ''') 82 | 83 | complexity = self.analyzer.analyze_complexity(test_file) 84 | self.assertTrue(complexity > 1) 85 | self.assertEqual(complexity, 4) # 4个条件分支 86 | 87 | def tearDown(self): 88 | """清理测试数据""" 89 | import shutil 90 | if os.path.exists(self.test_data_dir): 91 | shutil.rmtree(self.test_data_dir) 92 | 93 | if __name__ == '__main__': 94 | unittest.main() -------------------------------------------------------------------------------- /signatures.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "signatures": [ 3 | { 4 | "id": "CWE-78", 5 | "name": "OS命令注入", 6 | "description": "应用程序在构造操作系统命令时，未对用户输入进行适当的验证，可能导致命令注入攻击。", 7 | "severity": "高", 8 | "references": [ 9 | "https://cwe.mitre.org/data/definitions/78.html", 10 | "https://owasp.org/www-community/attacks/Command_Injection" 11 | ], 12 | "code_patterns": [ 13 | "os\\.system\\(.*\\)", 14 | "exec\\.Command\\(.*\\)", 15 | "shell\\.Run\\(.*\\)" 16 | ], 17 | "fix_suggestions": [ 18 | "使用参数化命令执行", 19 | "对用户输入进行严格的验证和过滤", 20 | "使用安全的API替代直接的命令执行" 21 | ] 22 | }, 23 | { 24 | "id": "CWE-89", 25 | "name": "SQL注入", 26 | "description": "应用程序在构造SQL查询时，未对用户输入进行适当的验证，可能导致SQL注入攻击。", 27 | "severity": "高", 28 | "references": [ 29 | "https://cwe.mitre.org/data/definitions/89.html", 30 | "https://owasp.org/www-community/attacks/SQL_Injection" 31 | ], 32 | "code_patterns": [ 33 | "db\\.Query\\(.*\\+.*\\)", 34 | "db\\.Exec\\(.*\\+.*\\)", 35 | "sql\\.Raw\\(.*\\)" 36 | ], 37 | "fix_suggestions": [ 38 | "使用参数化查询", 39 | "使用ORM框架", 40 | "对用户输入进行验证和转义" 41 | ] 42 | }, 43 | { 44 | "id": "CWE-200", 45 | "name": "敏感信息泄露", 46 | "description": "应用程序可能在日志、错误消息或响应中泄露敏感信息。", 47 | "severity": "中", 48 | "references": [ 49 | "https://cwe.mitre.org/data/definitions/200.html" 50 | ], 51 | "code_patterns": [ 52 | "log\\.Print\\(.*password.*\\)", 53 | "fmt\\.Printf\\(.*secret.*\\)", 54 | "\\.Debug\\(.*key.*\\)" 55 | ], 56 | "fix_suggestions": [ 57 | "避免记录敏感信息", 58 | "使用适当的日志级别", 59 | "实现敏感数据的脱敏处理" 60 | ] 61 | }, 62 | { 63 | "id": "CWE-22", 64 | "name": "路径遍历", 65 | "description": "应用程序在处理文件路径时，未对用户输入进行适当的验证，可能导致路径遍历攻击。", 66 | "severity": "高", 67 | "references": [ 68 | "https://cwe.mitre.org/data/definitions/22.html", 69 | "https://owasp.org/www-community/attacks/Path_Traversal" 70 | ], 71 | "code_patterns": [ 72 | "os\\.Open\\(.*\\)", 73 | "ioutil\\.ReadFile\\(.*\\)", 74 | "os\\.ReadFile\\(.*\\)" 75 | ], 76 | "fix_suggestions": [ 77 | "使用filepath.Clean()规范化路径", 78 | "限制文件操作在特定目录内", 79 | "验证文件路径不包含危险字符" 80 | ] 81 | }, 82 | { 83 | "id": "CWE-326", 84 | "name": "弱加密", 85 | "description": "应用程序使用了不安全或已过时的加密算法。", 86 | "severity": "中", 87 | "references": [ 88 | "https://cwe.mitre.org/data/definitions/326.html" 89 | ], 90 | "code_patterns": [ 91 | "md5\\.New\\(\\)", 92 | "sha1\\.New\\(\\)", 93 | "des\\.NewCipher\\(.*\\)" 94 | ], 95 | "fix_suggestions": [ 96 | "使用强加密算法（如AES）", 97 | "使用足够长度的密钥", 98 | "定期更新加密算法" 99 | ] 100 | } 101 | ] 102 | } -------------------------------------------------------------------------------- /go/internal/reporters/xml.go: -------------------------------------------------------------------------------- 1 | package reporters 2 | 3 | import ( 4 | "encoding/xml" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/re-movery/re-movery/internal/core" 9 | ) 10 | 11 | // XMLReporter is a reporter that generates XML reports 12 | type XMLReporter struct{} 13 | 14 | // NewXMLReporter creates a new XML reporter 15 | func NewXMLReporter() *XMLReporter { 16 | return &XMLReporter{} 17 | } 18 | 19 | // XMLReportData is the XML representation of the report data 20 | type XMLReportData struct { 21 | XMLName xml.Name `xml:"report"` 22 | Title string `xml:"title"` 23 | Timestamp string `xml:"timestamp"` 24 | Summary XMLSummary `xml:"summary"` 25 | Results []XMLFileResult `xml:"results>file"` 26 | } 27 | 28 | // XMLSummary is the XML representation of the summary 29 | type XMLSummary struct { 30 | TotalFiles int `xml:"totalFiles,attr"` 31 | High int `xml:"high,attr"` 32 | Medium int `xml:"medium,attr"` 33 | Low int `xml:"low,attr"` 34 | } 35 | 36 | // XMLFileResult is the XML representation of a file result 37 | type XMLFileResult struct { 38 | Path string `xml:"path,attr"` 39 | Matches []XMLMatch `xml:"match"` 40 | } 41 | 42 | // XMLMatch is the XML representation of a match 43 | type XMLMatch struct { 44 | ID string `xml:"id,attr"` 45 | Name string `xml:"name"` 46 | Severity string `xml:"severity"` 47 | Description string `xml:"description"` 48 | LineNumber int `xml:"lineNumber"` 49 | MatchedCode string `xml:"matchedCode"` 50 | Confidence float64 `xml:"confidence"` 51 | } 52 | 53 | // GenerateReport generates a report 54 | func (r *XMLReporter) GenerateReport(data core.ReportData, outputPath string) error { 55 | // Create output directory if it doesn't exist 56 | outputDir := filepath.Dir(outputPath) 57 | if err := os.MkdirAll(outputDir, 0755); err != nil { 58 | return err 59 | } 60 | 61 | // Create output file 62 | file, err := os.Create(outputPath) 63 | if err != nil { 64 | return err 65 | } 66 | defer file.Close() 67 | 68 | // Convert data to XML format 69 | xmlData := r.convertToXML(data) 70 | 71 | // Write XML header 72 | file.WriteString(xml.Header) 73 | 74 | // Marshal data to XML 75 | encoder := xml.NewEncoder(file) 76 | encoder.Indent("", " ") 77 | if err := encoder.Encode(xmlData); err != nil { 78 | return err 79 | } 80 | 81 | return nil 82 | } 83 | 84 | // convertToXML converts the report data to XML format 85 | func (r *XMLReporter) convertToXML(data core.ReportData) XMLReportData { 86 | xmlData := XMLReportData{ 87 | Title: data.Title, 88 | Timestamp: data.Timestamp, 89 | Summary: XMLSummary{ 90 | TotalFiles: data.Summary.TotalFiles, 91 | High: data.Summary.High, 92 | Medium: data.Summary.Medium, 93 | Low: data.Summary.Low, 94 | }, 95 | Results: []XMLFileResult{}, 96 | } 97 | 98 | // Convert results 99 | for filePath, matches := range data.Results { 100 | fileResult := XMLFileResult{ 101 | Path: filePath, 102 | Matches: []XMLMatch{}, 103 | } 104 | 105 | for _, match := range matches { 106 | xmlMatch := XMLMatch{ 107 | ID: match.Signature.ID, 108 | Name: match.Signature.Name, 109 | Severity: match.Signature.Severity, 110 | Description: match.Signature.Description, 111 | LineNumber: match.LineNumber, 112 | MatchedCode: match.MatchedCode, 113 | Confidence: match.Confidence, 114 | } 115 | fileResult.Matches = append(fileResult.Matches, xmlMatch) 116 | } 117 | 118 | xmlData.Results = append(xmlData.Results, fileResult) 119 | } 120 | 121 | return xmlData 122 | } -------------------------------------------------------------------------------- /src/config/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration settings for Movery 3 | """ 4 | import os 5 | from typing import Dict, List, Optional 6 | from dataclasses import dataclass 7 | 8 | @dataclass 9 | class ProcessingConfig: 10 | # Number of parallel processes to use 11 | num_processes: int = os.cpu_count() or 4 12 | 13 | # Memory settings 14 | max_memory_usage: int = 8 * 1024 * 1024 * 1024 # 8GB 15 | chunk_size: int = 1024 * 1024 # 1MB 16 | 17 | # Cache settings 18 | enable_cache: bool = True 19 | cache_dir: str = ".cache" 20 | cache_max_size: int = 1024 * 1024 * 1024 # 1GB 21 | 22 | # Language support 23 | supported_languages: List[str] = ["c", "cpp", "java", "python", "go", "javascript"] 24 | file_extensions: Dict[str, List[str]] = { 25 | "c": [".c", ".h"], 26 | "cpp": [".cpp", ".hpp", ".cc", ".hh"], 27 | "java": [".java"], 28 | "python": [".py"], 29 | "go": [".go"], 30 | "javascript": [".js", ".jsx", ".ts", ".tsx"] 31 | } 32 | 33 | @dataclass 34 | class DetectorConfig: 35 | # Vulnerability detection settings 36 | min_similarity: float = 0.8 37 | max_edit_distance: int = 10 38 | context_lines: int = 3 39 | 40 | # Analysis depth 41 | max_ast_depth: int = 50 42 | max_cfg_nodes: int = 1000 43 | 44 | # Pattern matching 45 | enable_semantic_match: bool = True 46 | enable_syntax_match: bool = True 47 | enable_token_match: bool = True 48 | 49 | # Reporting 50 | report_format: str = "html" 51 | report_dir: str = "reports" 52 | 53 | # Filtering 54 | exclude_patterns: List[str] = [ 55 | "**/test/*", 56 | "**/tests/*", 57 | "**/vendor/*", 58 | "**/node_modules/*" 59 | ] 60 | 61 | @dataclass 62 | class LoggingConfig: 63 | # Log settings 64 | log_level: str = "INFO" 65 | log_file: str = "movery.log" 66 | log_format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 67 | 68 | # Performance monitoring 69 | enable_profiling: bool = False 70 | profile_output: str = "profile.stats" 71 | 72 | # Progress reporting 73 | show_progress: bool = True 74 | progress_interval: int = 1 # seconds 75 | 76 | @dataclass 77 | class SecurityConfig: 78 | # Security settings 79 | max_file_size: int = 100 * 1024 * 1024 # 100MB 80 | allowed_schemes: List[str] = ["file", "http", "https"] 81 | enable_sandbox: bool = True 82 | sandbox_timeout: int = 60 # seconds 83 | 84 | # Access control 85 | require_auth: bool = False 86 | auth_token: Optional[str] = None 87 | 88 | # Rate limiting 89 | rate_limit: int = 100 # requests per minute 90 | rate_limit_period: int = 60 # seconds 91 | 92 | class MoveryConfig: 93 | def __init__(self): 94 | self.processing = ProcessingConfig() 95 | self.detector = DetectorConfig() 96 | self.logging = LoggingConfig() 97 | self.security = SecurityConfig() 98 | 99 | @classmethod 100 | def from_file(cls, config_file: str) -> "MoveryConfig": 101 | """Load configuration from file""" 102 | # TODO: Implement config file loading 103 | return cls() 104 | 105 | def to_file(self, config_file: str): 106 | """Save configuration to file""" 107 | # TODO: Implement config file saving 108 | pass 109 | 110 | def validate(self) -> bool: 111 | """Validate configuration settings""" 112 | # TODO: Add validation logic 113 | return True 114 | 115 | # Global configuration instance 116 | config = MoveryConfig() -------------------------------------------------------------------------------- /go/internal/core/config.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "gopkg.in/yaml.v3" 12 | ) 13 | 14 | // Config 表示应用程序配置 15 | type Config struct { 16 | Scanner ScannerConfig `json:"scanner" yaml:"scanner"` 17 | Web WebConfig `json:"web" yaml:"web"` 18 | Server ServerConfig `json:"server" yaml:"server"` 19 | } 20 | 21 | // ScannerConfig 表示扫描器配置 22 | type ScannerConfig struct { 23 | Parallel bool `json:"parallel" yaml:"parallel"` 24 | Incremental bool `json:"incremental" yaml:"incremental"` 25 | ConfidenceThreshold float64 `json:"confidenceThreshold" yaml:"confidenceThreshold"` 26 | ExcludePatterns []string `json:"excludePatterns" yaml:"excludePatterns"` 27 | } 28 | 29 | // WebConfig 表示Web界面配置 30 | type WebConfig struct { 31 | Host string `json:"host" yaml:"host"` 32 | Port int `json:"port" yaml:"port"` 33 | Debug bool `json:"debug" yaml:"debug"` 34 | } 35 | 36 | // ServerConfig 表示API服务器配置 37 | type ServerConfig struct { 38 | Host string `json:"host" yaml:"host"` 39 | Port int `json:"port" yaml:"port"` 40 | Debug bool `json:"debug" yaml:"debug"` 41 | } 42 | 43 | // NewConfig 创建一个新的配置对象，使用默认值 44 | func NewConfig() *Config { 45 | return &Config{ 46 | Scanner: ScannerConfig{ 47 | Parallel: false, 48 | Incremental: false, 49 | ConfidenceThreshold: 0.7, 50 | ExcludePatterns: []string{}, 51 | }, 52 | Web: WebConfig{ 53 | Host: "localhost", 54 | Port: 8080, 55 | Debug: false, 56 | }, 57 | Server: ServerConfig{ 58 | Host: "localhost", 59 | Port: 8081, 60 | Debug: false, 61 | }, 62 | } 63 | } 64 | 65 | // LoadConfig 从文件加载配置 66 | func LoadConfig(configPath string) (*Config, error) { 67 | // 如果未指定配置文件，则使用默认配置 68 | if configPath == "" { 69 | return NewConfig(), nil 70 | } 71 | 72 | // 检查文件是否存在 73 | if _, err := os.Stat(configPath); os.IsNotExist(err) { 74 | return nil, fmt.Errorf("配置文件不存在: %s", configPath) 75 | } 76 | 77 | // 读取文件内容 78 | data, err := ioutil.ReadFile(configPath) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | // 根据文件扩展名解析配置 84 | config := NewConfig() 85 | ext := strings.ToLower(filepath.Ext(configPath)) 86 | switch ext { 87 | case ".json": 88 | if err := json.Unmarshal(data, config); err != nil { 89 | return nil, err 90 | } 91 | case ".yaml", ".yml": 92 | if err := yaml.Unmarshal(data, config); err != nil { 93 | return nil, err 94 | } 95 | default: 96 | return nil, fmt.Errorf("不支持的配置文件格式: %s", ext) 97 | } 98 | 99 | return config, nil 100 | } 101 | 102 | // SaveConfig 将配置保存到文件 103 | func SaveConfig(config *Config, configPath string) error { 104 | // 创建输出目录（如果不存在） 105 | outputDir := filepath.Dir(configPath) 106 | if err := os.MkdirAll(outputDir, 0755); err != nil { 107 | return err 108 | } 109 | 110 | // 根据文件扩展名序列化配置 111 | var data []byte 112 | var err error 113 | ext := strings.ToLower(filepath.Ext(configPath)) 114 | switch ext { 115 | case ".json": 116 | data, err = json.MarshalIndent(config, "", " ") 117 | if err != nil { 118 | return err 119 | } 120 | case ".yaml", ".yml": 121 | data, err = yaml.Marshal(config) 122 | if err != nil { 123 | return err 124 | } 125 | default: 126 | return fmt.Errorf("不支持的配置文件格式: %s", ext) 127 | } 128 | 129 | // 写入文件 130 | return ioutil.WriteFile(configPath, data, 0644) 131 | } 132 | 133 | // ApplyToScanner 将配置应用到扫描器 134 | func (c *Config) ApplyToScanner(scanner *Scanner) { 135 | scanner.SetParallel(c.Scanner.Parallel) 136 | scanner.SetIncremental(c.Scanner.Incremental) 137 | scanner.SetConfidenceThreshold(c.Scanner.ConfidenceThreshold) 138 | } -------------------------------------------------------------------------------- /go/internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/spf13/viper" 5 | ) 6 | 7 | // Config represents the application configuration 8 | type Config struct { 9 | Processing ProcessingConfig `mapstructure:"processing"` 10 | Detector DetectorConfig `mapstructure:"detector"` 11 | Logging LoggingConfig `mapstructure:"logging"` 12 | Security SecurityConfig `mapstructure:"security"` 13 | } 14 | 15 | // ProcessingConfig contains processing-related configuration 16 | type ProcessingConfig struct { 17 | NumWorkers int `mapstructure:"num_workers"` 18 | MaxMemoryGB float64 `mapstructure:"max_memory_gb"` 19 | ChunkSizeMB int `mapstructure:"chunk_size_mb"` 20 | EnableCache bool `mapstructure:"enable_cache"` 21 | CacheSize int `mapstructure:"cache_size"` 22 | Languages []string `mapstructure:"languages"` 23 | } 24 | 25 | // DetectorConfig contains detector-related configuration 26 | type DetectorConfig struct { 27 | MinSimilarity float64 `mapstructure:"min_similarity"` 28 | EditDistance int `mapstructure:"edit_distance"` 29 | ContextLines int `mapstructure:"context_lines"` 30 | ASTDepth int `mapstructure:"ast_depth"` 31 | CFGNodes int `mapstructure:"cfg_nodes"` 32 | ReportFormat []string `mapstructure:"report_format"` 33 | ExcludePatterns []string `mapstructure:"exclude_patterns"` 34 | } 35 | 36 | // LoggingConfig contains logging-related configuration 37 | type LoggingConfig struct { 38 | Level string `mapstructure:"level"` 39 | File string `mapstructure:"file"` 40 | Format string `mapstructure:"format"` 41 | EnableProfiling bool `mapstructure:"enable_profiling"` 42 | ShowProgress bool `mapstructure:"show_progress"` 43 | } 44 | 45 | // SecurityConfig contains security-related configuration 46 | type SecurityConfig struct { 47 | MaxFileSizeMB int `mapstructure:"max_file_size_mb"` 48 | AllowedSchemes []string `mapstructure:"allowed_schemes"` 49 | EnableSandbox bool `mapstructure:"enable_sandbox"` 50 | RequireAuth bool `mapstructure:"require_auth"` 51 | RateLimitPerHour int `mapstructure:"rate_limit_per_hour"` 52 | } 53 | 54 | // LoadConfig loads the configuration from file 55 | func LoadConfig(configFile string) (*Config, error) { 56 | viper.SetConfigFile(configFile) 57 | viper.SetConfigType("json") 58 | 59 | if err := viper.ReadInConfig(); err != nil { 60 | return nil, err 61 | } 62 | 63 | var config Config 64 | if err := viper.Unmarshal(&config); err != nil { 65 | return nil, err 66 | } 67 | 68 | return &config, nil 69 | } 70 | 71 | // SetDefaults sets default configuration values 72 | func SetDefaults() { 73 | viper.SetDefault("processing.num_workers", 4) 74 | viper.SetDefault("processing.max_memory_gb", 8) 75 | viper.SetDefault("processing.chunk_size_mb", 1) 76 | viper.SetDefault("processing.enable_cache", true) 77 | viper.SetDefault("processing.cache_size", 1000) 78 | viper.SetDefault("processing.languages", []string{"go", "java", "python", "javascript"}) 79 | 80 | viper.SetDefault("detector.min_similarity", 0.8) 81 | viper.SetDefault("detector.edit_distance", 3) 82 | viper.SetDefault("detector.context_lines", 3) 83 | viper.SetDefault("detector.ast_depth", 5) 84 | viper.SetDefault("detector.cfg_nodes", 100) 85 | viper.SetDefault("detector.report_format", []string{"html", "json"}) 86 | 87 | viper.SetDefault("logging.level", "info") 88 | viper.SetDefault("logging.format", "text") 89 | viper.SetDefault("logging.enable_profiling", false) 90 | viper.SetDefault("logging.show_progress", true) 91 | 92 | viper.SetDefault("security.max_file_size_mb", 10) 93 | viper.SetDefault("security.enable_sandbox", true) 94 | viper.SetDefault("security.require_auth", false) 95 | viper.SetDefault("security.rate_limit_per_hour", 1000) 96 | } -------------------------------------------------------------------------------- /go/internal/web/app.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/re-movery/re-movery/internal/core" 14 | "github.com/re-movery/re-movery/internal/detectors" 15 | ) 16 | 17 | // App is the web application 18 | type App struct { 19 | scanner *core.Scanner 20 | router *gin.Engine 21 | } 22 | 23 | // NewApp creates a new web application 24 | func NewApp() *App { 25 | app := &App{ 26 | scanner: core.NewScanner(), 27 | router: gin.Default(), 28 | } 29 | 30 | // Register detectors 31 | app.scanner.RegisterDetector(detectors.NewPythonDetector()) 32 | app.scanner.RegisterDetector(detectors.NewJavaScriptDetector()) 33 | 34 | // Setup routes 35 | app.setupRoutes() 36 | 37 | return app 38 | } 39 | 40 | // setupRoutes sets up the routes for the web application 41 | func (a *App) setupRoutes() { 42 | // Serve static files 43 | a.router.Static("/static", "./static") 44 | 45 | // Load templates 46 | a.router.LoadHTMLGlob("templates/*") 47 | 48 | // Routes 49 | a.router.GET("/", a.indexHandler) 50 | a.router.POST("/scan/file", a.scanFileHandler) 51 | a.router.POST("/scan/directory", a.scanDirectoryHandler) 52 | a.router.GET("/api/languages", a.languagesHandler) 53 | a.router.GET("/health", a.healthHandler) 54 | } 55 | 56 | // Run runs the web application 57 | func (a *App) Run(host string, port int) error { 58 | return a.router.Run(fmt.Sprintf("%s:%d", host, port)) 59 | } 60 | 61 | // indexHandler handles the index page 62 | func (a *App) indexHandler(c *gin.Context) { 63 | c.HTML(http.StatusOK, "index.html", gin.H{ 64 | "title": "Re-movery - Security Scanner", 65 | }) 66 | } 67 | 68 | // scanFileHandler handles file scanning 69 | func (a *App) scanFileHandler(c *gin.Context) { 70 | // Get file from form 71 | file, err := c.FormFile("file") 72 | if err != nil { 73 | c.JSON(http.StatusBadRequest, gin.H{ 74 | "error": "No file provided", 75 | }) 76 | return 77 | } 78 | 79 | // Save file to temporary location 80 | tempFile := filepath.Join(os.TempDir(), file.Filename) 81 | if err := c.SaveUploadedFile(file, tempFile); err != nil { 82 | c.JSON(http.StatusInternalServerError, gin.H{ 83 | "error": "Failed to save file", 84 | }) 85 | return 86 | } 87 | defer os.Remove(tempFile) 88 | 89 | // Scan file 90 | results, err := a.scanner.ScanFile(tempFile) 91 | if err != nil { 92 | c.JSON(http.StatusInternalServerError, gin.H{ 93 | "error": fmt.Sprintf("Failed to scan file: %v", err), 94 | }) 95 | return 96 | } 97 | 98 | // Generate summary 99 | summary := core.GenerateSummary(map[string][]core.Match{ 100 | file.Filename: results, 101 | }) 102 | 103 | // Return results 104 | c.JSON(http.StatusOK, gin.H{ 105 | "results": map[string][]core.Match{ 106 | file.Filename: results, 107 | }, 108 | "summary": summary, 109 | }) 110 | } 111 | 112 | // scanDirectoryHandler handles directory scanning 113 | func (a *App) scanDirectoryHandler(c *gin.Context) { 114 | // Get directory path from form 115 | directory := c.PostForm("directory") 116 | if directory == "" { 117 | c.JSON(http.StatusBadRequest, gin.H{ 118 | "error": "No directory provided", 119 | }) 120 | return 121 | } 122 | 123 | // Check if directory exists 124 | if _, err := os.Stat(directory); os.IsNotExist(err) { 125 | c.JSON(http.StatusBadRequest, gin.H{ 126 | "error": "Directory does not exist", 127 | }) 128 | return 129 | } 130 | 131 | // Get exclude patterns 132 | excludePatterns := c.PostFormArray("exclude") 133 | 134 | // Scan directory 135 | results, err := a.scanner.ScanDirectory(directory, excludePatterns) 136 | if err != nil { 137 | c.JSON(http.StatusInternalServerError, gin.H{ 138 | "error": fmt.Sprintf("Failed to scan directory: %v", err), 139 | }) 140 | return 141 | } 142 | 143 | // Generate summary 144 | summary := core.GenerateSummary(results) 145 | 146 | // Return results 147 | c.JSON(http.StatusOK, gin.H{ 148 | "results": results, 149 | "summary": summary, 150 | }) 151 | } 152 | 153 | // languagesHandler handles the supported languages request 154 | func (a *App) languagesHandler(c *gin.Context) { 155 | languages := a.scanner.SupportedLanguages() 156 | c.JSON(http.StatusOK, gin.H{ 157 | "languages": languages, 158 | }) 159 | } 160 | 161 | // healthHandler handles the health check request 162 | func (a *App) healthHandler(c *gin.Context) { 163 | c.JSON(http.StatusOK, gin.H{ 164 | "status": "ok", 165 | "time": time.Now().Format(time.RFC3339), 166 | }) 167 | } -------------------------------------------------------------------------------- /go/web/templates/report.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Re-movery Vulnerability Report 7 | 8 | 9 | 29 | 30 | 31 |
32 |

Re-movery Vulnerability Report

33 | 34 |
35 |
36 |
37 |
38 |
Report Summary
39 |

Generated at: {{.GeneratedAt}}

40 |

Total Files Scanned: {{.TotalFiles}}

41 |

Total Vulnerabilities Found: {{.TotalMatches}}

42 |
43 |
44 |
45 |
46 | 47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 | 64 |

Detailed Findings

65 | {{range .Vulnerabilities}} 66 |
67 |
68 |
69 | {{.Signature.Name}} 70 | 71 | {{.Signature.Severity}} 72 | 73 |
74 |
75 |
76 |
ID: {{.Signature.ID}}
77 |

{{.Signature.Description}}

78 | 79 |
80 | File: {{.File}}
81 | Line: {{.Line}}
82 | Confidence: {{printf "%.1f%%" (mul .Confidence 100)}} 83 |
84 | 85 |
86 | 
{{.Code}}
87 |
88 | 89 | {{if .Context}} 90 |
91 |
Context:
92 |
93 | 
{{range .Context}}{{.}}
 94 | {{end}}
95 |
96 |
97 | {{end}} 98 | 99 | {{if .Signature.References}} 100 |
101 |
References:
102 |
    103 | {{range .Signature.References}} 104 |
  • {{.}}
    • 105 | {{end}} 106 |
107 |
108 | {{end}} 109 |
110 |
111 | {{end}} 112 |
113 | 114 | 129 | 130 | -------------------------------------------------------------------------------- /go/internal/core/config_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | // 测试创建新配置 13 | func TestNewConfig(t *testing.T) { 14 | config := NewConfig() 15 | assert.NotNil(t, config) 16 | 17 | // 检查默认值 18 | assert.False(t, config.Scanner.Parallel) 19 | assert.False(t, config.Scanner.Incremental) 20 | assert.Equal(t, 0.7, config.Scanner.ConfidenceThreshold) 21 | assert.Equal(t, "localhost", config.Web.Host) 22 | assert.Equal(t, 8080, config.Web.Port) 23 | assert.False(t, config.Web.Debug) 24 | assert.Equal(t, "localhost", config.Server.Host) 25 | assert.Equal(t, 8081, config.Server.Port) 26 | assert.False(t, config.Server.Debug) 27 | } 28 | 29 | // 测试加载JSON配置 30 | func TestLoadConfigJSON(t *testing.T) { 31 | // 创建临时配置文件 32 | content := []byte(`{ 33 | "scanner": { 34 | "parallel": true, 35 | "incremental": true, 36 | "confidenceThreshold": 0.8, 37 | "excludePatterns": ["node_modules", "*.min.js"] 38 | }, 39 | "web": { 40 | "host": "0.0.0.0", 41 | "port": 9090, 42 | "debug": true 43 | }, 44 | "server": { 45 | "host": "0.0.0.0", 46 | "port": 9091, 47 | "debug": true 48 | } 49 | }`) 50 | 51 | tmpfile, err := ioutil.TempFile("", "config-*.json") 52 | assert.NoError(t, err) 53 | defer os.Remove(tmpfile.Name()) 54 | 55 | _, err = tmpfile.Write(content) 56 | assert.NoError(t, err) 57 | err = tmpfile.Close() 58 | assert.NoError(t, err) 59 | 60 | // 加载配置 61 | config, err := LoadConfig(tmpfile.Name()) 62 | assert.NoError(t, err) 63 | assert.NotNil(t, config) 64 | 65 | // 检查加载的值 66 | assert.True(t, config.Scanner.Parallel) 67 | assert.True(t, config.Scanner.Incremental) 68 | assert.Equal(t, 0.8, config.Scanner.ConfidenceThreshold) 69 | assert.Equal(t, []string{"node_modules", "*.min.js"}, config.Scanner.ExcludePatterns) 70 | assert.Equal(t, "0.0.0.0", config.Web.Host) 71 | assert.Equal(t, 9090, config.Web.Port) 72 | assert.True(t, config.Web.Debug) 73 | assert.Equal(t, "0.0.0.0", config.Server.Host) 74 | assert.Equal(t, 9091, config.Server.Port) 75 | assert.True(t, config.Server.Debug) 76 | } 77 | 78 | // 测试加载YAML配置 79 | func TestLoadConfigYAML(t *testing.T) { 80 | // 创建临时配置文件 81 | content := []byte(`scanner: 82 | parallel: true 83 | incremental: true 84 | confidenceThreshold: 0.8 85 | excludePatterns: 86 | - node_modules 87 | - "*.min.js" 88 | web: 89 | host: 0.0.0.0 90 | port: 9090 91 | debug: true 92 | server: 93 | host: 0.0.0.0 94 | port: 9091 95 | debug: true 96 | `) 97 | 98 | tmpfile, err := ioutil.TempFile("", "config-*.yaml") 99 | assert.NoError(t, err) 100 | defer os.Remove(tmpfile.Name()) 101 | 102 | _, err = tmpfile.Write(content) 103 | assert.NoError(t, err) 104 | err = tmpfile.Close() 105 | assert.NoError(t, err) 106 | 107 | // 加载配置 108 | config, err := LoadConfig(tmpfile.Name()) 109 | assert.NoError(t, err) 110 | assert.NotNil(t, config) 111 | 112 | // 检查加载的值 113 | assert.True(t, config.Scanner.Parallel) 114 | assert.True(t, config.Scanner.Incremental) 115 | assert.Equal(t, 0.8, config.Scanner.ConfidenceThreshold) 116 | assert.Equal(t, []string{"node_modules", "*.min.js"}, config.Scanner.ExcludePatterns) 117 | assert.Equal(t, "0.0.0.0", config.Web.Host) 118 | assert.Equal(t, 9090, config.Web.Port) 119 | assert.True(t, config.Web.Debug) 120 | assert.Equal(t, "0.0.0.0", config.Server.Host) 121 | assert.Equal(t, 9091, config.Server.Port) 122 | assert.True(t, config.Server.Debug) 123 | } 124 | 125 | // 测试保存配置 126 | func TestSaveConfig(t *testing.T) { 127 | // 创建配置 128 | config := NewConfig() 129 | config.Scanner.Parallel = true 130 | config.Scanner.Incremental = true 131 | config.Scanner.ConfidenceThreshold = 0.8 132 | config.Scanner.ExcludePatterns = []string{"node_modules", "*.min.js"} 133 | config.Web.Host = "0.0.0.0" 134 | config.Web.Port = 9090 135 | config.Web.Debug = true 136 | config.Server.Host = "0.0.0.0" 137 | config.Server.Port = 9091 138 | config.Server.Debug = true 139 | 140 | // 创建临时文件路径 141 | tmpdir, err := ioutil.TempDir("", "config-test") 142 | assert.NoError(t, err) 143 | defer os.RemoveAll(tmpdir) 144 | 145 | // 保存JSON配置 146 | jsonPath := filepath.Join(tmpdir, "config.json") 147 | err = SaveConfig(config, jsonPath) 148 | assert.NoError(t, err) 149 | 150 | // 保存YAML配置 151 | yamlPath := filepath.Join(tmpdir, "config.yaml") 152 | err = SaveConfig(config, yamlPath) 153 | assert.NoError(t, err) 154 | 155 | // 重新加载JSON配置 156 | jsonConfig, err := LoadConfig(jsonPath) 157 | assert.NoError(t, err) 158 | assert.Equal(t, config, jsonConfig) 159 | 160 | // 重新加载YAML配置 161 | yamlConfig, err := LoadConfig(yamlPath) 162 | assert.NoError(t, err) 163 | assert.Equal(t, config, yamlConfig) 164 | } 165 | 166 | // 测试应用配置到扫描器 167 | func TestApplyToScanner(t *testing.T) { 168 | // 创建配置 169 | config := NewConfig() 170 | config.Scanner.Parallel = true 171 | config.Scanner.Incremental = true 172 | config.Scanner.ConfidenceThreshold = 0.8 173 | 174 | // 创建扫描器 175 | scanner := NewScanner() 176 | 177 | // 应用配置 178 | config.ApplyToScanner(scanner) 179 | 180 | // 检查扫描器设置 181 | assert.True(t, scanner.IsParallel()) 182 | assert.True(t, scanner.IsIncremental()) 183 | assert.Equal(t, 0.8, scanner.confidenceThreshold) 184 | } -------------------------------------------------------------------------------- /movery/tests/unit/test_vulnerability.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import json 4 | import tempfile 5 | import shutil 6 | import ast 7 | from movery.detectors.vulnerability import VulnerabilityDetector, Signature, VulnerabilityMatch 8 | 9 | class TestVulnerabilityDetector(unittest.TestCase): 10 | def setUp(self): 11 | """测试前的准备工作""" 12 | self.detector = VulnerabilityDetector() 13 | self.test_dir = tempfile.mkdtemp() 14 | 15 | # 创建测试签名文件 16 | self.signatures = { 17 | "signatures": [ 18 | { 19 | "id": "CMD001", 20 | "name": "命令注入", 21 | "severity": "high", 22 | "code_patterns": [ 23 | "os\\.system\\([^)]*\\)", 24 | "subprocess\\.call\\([^)]*\\)" 25 | ] 26 | }, 27 | { 28 | "id": "SQL001", 29 | "name": "SQL注入", 30 | "severity": "high", 31 | "code_patterns": [ 32 | "execute\\(['\"][^'\"]*%[^'\"]*['\"]\\)", 33 | "executemany\\(['\"][^'\"]*%[^'\"]*['\"]\\)" 34 | ] 35 | } 36 | ] 37 | } 38 | 39 | self.signature_file = os.path.join(self.test_dir, "signatures.json") 40 | with open(self.signature_file, "w") as f: 41 | json.dump(self.signatures, f) 42 | 43 | # 创建测试代码文件 44 | self.test_code = ''' 45 | import os 46 | import subprocess 47 | 48 | def unsafe_command(): 49 | cmd = "ls -l" 50 | os.system(cmd) 51 | subprocess.call(["echo", "hello"]) 52 | 53 | def unsafe_sql(): 54 | query = "SELECT * FROM users WHERE id = %s" 55 | cursor.execute(query % user_id) 56 | ''' 57 | self.test_file = os.path.join(self.test_dir, "test_code.py") 58 | with open(self.test_file, "w") as f: 59 | f.write(self.test_code) 60 | 61 | def tearDown(self): 62 | """测试后的清理工作""" 63 | shutil.rmtree(self.test_dir) 64 | 65 | def test_load_signatures(self): 66 | """测试加载签名文件""" 67 | self.detector.load_signatures(self.signature_file) 68 | 69 | self.assertEqual(len(self.detector.signatures), 2) 70 | self.assertEqual(self.detector.signatures[0].id, "CMD001") 71 | self.assertEqual(self.detector.signatures[0].name, "命令注入") 72 | self.assertEqual(len(self.detector.signatures[0].code_patterns), 2) 73 | 74 | def test_detect_file(self): 75 | """测试文件漏洞检测""" 76 | self.detector.load_signatures(self.signature_file) 77 | matches = self.detector.detect_file(self.test_file) 78 | 79 | self.assertGreater(len(matches), 0) 80 | for match in matches: 81 | self.assertIsInstance(match, VulnerabilityMatch) 82 | self.assertIsInstance(match.signature, Signature) 83 | self.assertGreater(match.confidence, 0.7) 84 | 85 | def test_analyze_ast(self): 86 | """测试AST分析""" 87 | self.detector.load_signatures(self.signature_file) 88 | with open(self.test_file, 'r') as f: 89 | tree = ast.parse(f.read()) 90 | matches = self.detector.analyze_ast(tree) 91 | 92 | self.assertGreater(len(matches), 0) 93 | for match in matches: 94 | self.assertIsInstance(match, VulnerabilityMatch) 95 | self.assertGreater(match.line_number, 0) 96 | 97 | def test_detect_similar_patterns(self): 98 | """测试相似模式检测""" 99 | similar_code = ''' 100 | import os 101 | import subprocess 102 | 103 | def custom_system(cmd): 104 | os.system(cmd) # 直接模式 105 | 106 | def modified_system(command): 107 | os.system(command) # 相似模式 108 | ''' 109 | similar_file = os.path.join(self.test_dir, "similar_code.py") 110 | with open(similar_file, "w") as f: 111 | f.write(similar_code) 112 | 113 | self.detector.load_signatures(self.signature_file) 114 | matches = self.detector.detect_similar_patterns(similar_code) 115 | 116 | self.assertGreater(len(matches), 0) 117 | for match in matches: 118 | self.assertIsInstance(match, VulnerabilityMatch) 119 | self.assertGreater(match.confidence, 0.8) 120 | 121 | def test_calculate_confidence(self): 122 | """测试置信度计算""" 123 | test_cases = [ 124 | ("os.system('ls')", r"os\.system\([^)]*\)", 0.8), 125 | ("subprocess.call(['ls'])", r"subprocess\.call\([^)]*\)", 0.9), 126 | ("import os; os.system('ls')", r"os\.system\([^)]*\)", 1.0) 127 | ] 128 | 129 | for code, pattern, expected in test_cases: 130 | confidence = self.detector._calculate_confidence(code, pattern) 131 | self.assertGreaterEqual(confidence, expected) 132 | self.assertLessEqual(confidence, 1.0) 133 | 134 | def test_calculate_similarity(self): 135 | """测试相似度计算""" 136 | test_cases = [ 137 | ("os.system", "os.system", 1.0), 138 | ("os.system", "subprocess.system", 0.5), 139 | ("execute", "executemany", 0.7) 140 | ] 141 | 142 | for str1, str2, expected in test_cases: 143 | similarity = self.detector._calculate_similarity(str1, str2) 144 | self.assertGreaterEqual(similarity, expected - 0.1) 145 | self.assertLessEqual(similarity, 1.0) 146 | 147 | if __name__ == '__main__': 148 | unittest.main() -------------------------------------------------------------------------------- /movery/tests/security/test_security.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import sys 4 | import tempfile 5 | import shutil 6 | import subprocess 7 | sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 8 | 9 | from movery.detectors.vulnerability import VulnerabilityDetector 10 | from movery.utils.security import SecurityChecker 11 | 12 | class TestSecurity(unittest.TestCase): 13 | def setUp(self): 14 | """设置测试环境""" 15 | self.test_dir = tempfile.mkdtemp() 16 | self.security_checker = SecurityChecker() 17 | self.detector = VulnerabilityDetector() 18 | 19 | def create_test_file(self, content): 20 | """创建测试文件""" 21 | file_path = os.path.join(self.test_dir, 'test_file.py') 22 | with open(file_path, 'w') as f: 23 | f.write(content) 24 | return file_path 25 | 26 | def test_memory_limit(self): 27 | """测试内存限制""" 28 | # 创建一个可能导致内存溢出的文件 29 | test_file = self.create_test_file(''' 30 | def memory_intensive(): 31 | large_list = [i for i in range(10**8)] # 尝试创建大列表 32 | return large_list 33 | ''') 34 | 35 | # 检查内存使用 36 | memory_usage = self.security_checker.check_memory_usage(test_file) 37 | self.assertLess(memory_usage, 8 * 1024 * 1024 * 1024) # 8GB限制 38 | 39 | def test_execution_timeout(self): 40 | """测试执行超时""" 41 | # 创建一个可能导致无限循环的文件 42 | test_file = self.create_test_file(''' 43 | def infinite_loop(): 44 | while True: 45 | pass 46 | ''') 47 | 48 | # 检查执行时间 49 | with self.assertRaises(TimeoutError): 50 | self.security_checker.check_execution_time(test_file, timeout=5) 51 | 52 | def test_file_access(self): 53 | """测试文件访问限制""" 54 | # 创建测试文件 55 | test_file = self.create_test_file(''' 56 | import os 57 | 58 | def access_sensitive_file(): 59 | with open('/etc/passwd', 'r') as f: 60 | return f.read() 61 | ''') 62 | 63 | # 检查文件访问 64 | violations = self.security_checker.check_file_access(test_file) 65 | self.assertTrue(len(violations) > 0) 66 | self.assertIn('/etc/passwd', violations[0]) 67 | 68 | def test_network_access(self): 69 | """测试网络访问限制""" 70 | # 创建测试文件 71 | test_file = self.create_test_file(''' 72 | import socket 73 | 74 | def connect_external(): 75 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 76 | sock.connect(('example.com', 80)) 77 | ''') 78 | 79 | # 检查网络访问 80 | violations = self.security_checker.check_network_access(test_file) 81 | self.assertTrue(len(violations) > 0) 82 | self.assertIn('socket.connect', violations[0]) 83 | 84 | def test_code_injection(self): 85 | """测试代码注入防护""" 86 | # 创建测试文件 87 | test_file = self.create_test_file(''' 88 | def execute_input(user_input): 89 | exec(user_input) # 危险的代码执行 90 | ''') 91 | 92 | # 检查代码注入 93 | vulnerabilities = self.detector.detect_file(test_file) 94 | self.assertTrue(len(vulnerabilities) > 0) 95 | self.assertEqual(vulnerabilities[0].severity, 'HIGH') 96 | 97 | def test_input_validation(self): 98 | """测试输入验证""" 99 | # 创建测试文件 100 | test_file = self.create_test_file(''' 101 | def process_input(user_input): 102 | # 没有验证的输入处理 103 | return eval(user_input) 104 | ''') 105 | 106 | # 检查输入验证 107 | issues = self.security_checker.check_input_validation(test_file) 108 | self.assertTrue(len(issues) > 0) 109 | self.assertIn('eval', str(issues[0])) 110 | 111 | def test_secure_random(self): 112 | """测试安全随机数生成""" 113 | # 创建测试文件 114 | test_file = self.create_test_file(''' 115 | import random 116 | 117 | def generate_token(): 118 | return ''.join(random.choice('0123456789ABCDEF') for i in range(32)) 119 | ''') 120 | 121 | # 检查随机数生成 122 | issues = self.security_checker.check_random_generation(test_file) 123 | self.assertTrue(len(issues) > 0) 124 | self.assertIn('random.choice', str(issues[0])) 125 | 126 | def test_sensitive_data(self): 127 | """测试敏感数据处理""" 128 | # 创建测试文件 129 | test_file = self.create_test_file(''' 130 | def process_password(password): 131 | print(f"Password is: {password}") # 敏感信息泄露 132 | return hash(password) # 不安全的哈希 133 | ''') 134 | 135 | # 检查敏感数据处理 136 | issues = self.security_checker.check_sensitive_data(test_file) 137 | self.assertTrue(len(issues) > 0) 138 | self.assertIn('password', str(issues[0]).lower()) 139 | 140 | def test_sandbox_escape(self): 141 | """测试沙箱逃逸防护""" 142 | # 创建测试文件 143 | test_file = self.create_test_file(''' 144 | import subprocess 145 | import os 146 | 147 | def dangerous_operation(): 148 | os.system('rm -rf /') # 危险的系统命令 149 | subprocess.call(['chmod', '777', '/etc/passwd']) # 危险的权限修改 150 | ''') 151 | 152 | # 检查沙箱逃逸 153 | violations = self.security_checker.check_sandbox_escape(test_file) 154 | self.assertTrue(len(violations) > 0) 155 | self.assertIn('os.system', str(violations[0])) 156 | 157 | def tearDown(self): 158 | """清理测试环境""" 159 | shutil.rmtree(self.test_dir) 160 | 161 | if __name__ == '__main__': 162 | unittest.main() -------------------------------------------------------------------------------- /signatures.json: -------------------------------------------------------------------------------- 1 | { 2 | "signatures": [ 3 | { 4 | "id": "CWE-78", 5 | "name": "OS Command Injection", 6 | "description": "The software constructs all or part of an OS command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended OS command when it is sent to a downstream component.", 7 | "severity": "CRITICAL", 8 | "cwe_id": "CWE-78", 9 | "affected_languages": ["python", "php", "javascript"], 10 | "code_patterns": [ 11 | "os\\.system\\(.*\\)", 12 | "subprocess\\.call\\(.*shell\\s*=\\s*True.*\\)", 13 | "exec\\(.*\\)", 14 | "eval\\(.*\\)" 15 | ], 16 | "fix_patterns": [ 17 | "shlex.quote(command)", 18 | "subprocess.run([command], shell=False)", 19 | "ast.literal_eval(input)" 20 | ], 21 | "context_patterns": [ 22 | "import\\s+os", 23 | "import\\s+subprocess", 24 | "import\\s+shlex" 25 | ] 26 | }, 27 | { 28 | "id": "CWE-89", 29 | "name": "SQL Injection", 30 | "description": "The software constructs all or part of an SQL command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended SQL command when it is sent to a downstream component.", 31 | "severity": "CRITICAL", 32 | "cwe_id": "CWE-89", 33 | "affected_languages": ["python", "php", "java"], 34 | "code_patterns": [ 35 | "cursor\\.execute\\(.*%.*\\)", 36 | "cursor\\.execute\\(.*\\+.*\\)", 37 | "cursor\\.executemany\\(.*%.*\\)", 38 | "mysql_query\\(.*\\$.*\\)" 39 | ], 40 | "fix_patterns": [ 41 | "cursor.execute(query, params)", 42 | "cursor.executemany(query, params)", 43 | "prepared_statement.setString(1, input)" 44 | ], 45 | "context_patterns": [ 46 | "import\\s+sqlite3", 47 | "import\\s+mysql", 48 | "import\\s+psycopg2" 49 | ] 50 | }, 51 | { 52 | "id": "CWE-22", 53 | "name": "Path Traversal", 54 | "description": "The software uses external input to construct a pathname that is intended to identify a file or directory that is located underneath a restricted parent directory, but the software does not properly neutralize special elements within the pathname that can cause the pathname to resolve to a location that is outside of the restricted directory.", 55 | "severity": "HIGH", 56 | "cwe_id": "CWE-22", 57 | "affected_languages": ["python", "php", "java", "javascript"], 58 | "code_patterns": [ 59 | "open\\(.*\\+.*\\)", 60 | "file_get_contents\\(.*\\$.*\\)", 61 | "new\\s+File\\(.*\\+.*\\)" 62 | ], 63 | "fix_patterns": [ 64 | "os.path.abspath(os.path.join(base_dir, filename))", 65 | "os.path.normpath(path)", 66 | "Path(path).resolve().is_relative_to(base_dir)" 67 | ], 68 | "context_patterns": [ 69 | "import\\s+os", 70 | "from\\s+pathlib\\s+import\\s+Path" 71 | ] 72 | }, 73 | { 74 | "id": "CWE-79", 75 | "name": "Cross-site Scripting (XSS)", 76 | "description": "The software does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users.", 77 | "severity": "HIGH", 78 | "cwe_id": "CWE-79", 79 | "affected_languages": ["python", "php", "javascript"], 80 | "code_patterns": [ 81 | "innerHTML\\s*=.*", 82 | "document\\.write\\(.*\\)", 83 | "\\$\\(.*\\)\\.html\\(.*\\)" 84 | ], 85 | "fix_patterns": [ 86 | "textContent = content", 87 | "innerText = content", 88 | "createElement('div')" 89 | ], 90 | "context_patterns": [ 91 | "