├── stack.yaml ├── cabal.project ├── kythe-schema ├── Setup.hs ├── src │ └── Language │ │ └── Kythe │ │ └── Schema │ │ ├── Raw │ │ ├── VName.hs │ │ └── Proto.hs │ │ ├── Raw.hs │ │ └── Typed.hs └── kythe-schema.cabal ├── text-offset ├── Setup.hs ├── text-offset.cabal ├── tests │ └── OffsetTest.hs └── src │ └── Data │ └── Text │ └── Offset.hs ├── haskell-indexer-pathutil ├── Setup.hs ├── haskell-indexer-pathutil.cabal └── src │ └── Language │ └── Haskell │ └── Indexer │ └── Util │ └── Path.hs ├── haskell-indexer-plugin ├── Setup.hs ├── CHANGELOG.md ├── haskell-indexer-plugin.cabal └── src │ └── Haskell │ └── Indexer │ └── Plugin.hs ├── haskell-indexer-translate ├── Setup.hs ├── haskell-indexer-translate.cabal └── src │ └── Language │ └── Haskell │ └── Indexer │ ├── Translate │ ├── Utils.hs │ └── Render.hs │ └── Translate.hs ├── haskell-indexer-backend-core ├── Setup.hs ├── haskell-indexer-backend-core.cabal └── src │ └── Language │ └── Haskell │ └── Indexer │ └── Backend │ └── AnalysisOptions.hs ├── haskell-indexer-backend-ghc ├── Setup.hs ├── testdata │ ├── basic │ │ ├── ffi.c │ │ ├── ExecutableMain.hs │ │ ├── UsedByTH.hs │ │ ├── ImportRefsHiding.hs │ │ ├── DataConWrap.hs │ │ ├── Imports.hs │ │ ├── Utf8.hs │ │ ├── TypeVsTerm.hs │ │ ├── DocUri.hs │ │ ├── DummyInclude.hs │ │ ├── FunCall.hs │ │ ├── ArgRef.hs │ │ ├── ForeignImport.hs │ │ ├── TypeOperators.hs │ │ ├── CppInclude.hs │ │ ├── TypeClassRef.hs │ │ ├── ForeignExport.hs │ │ ├── DataDecl.hs │ │ ├── TemplateHaskellCodeExec.hs │ │ ├── TemplateHaskellQuotation.hs │ │ ├── PatSyn.hs │ │ ├── TemplateHaskellCodeExecFFI.hs │ │ ├── TypeClass.hs │ │ ├── ImpExpDefs.hs │ │ ├── ImportRefs.hs │ │ ├── RecursiveRef.hs │ │ ├── RecordRead.hs │ │ ├── LocalRef.hs │ │ └── RecordWrite.hs │ └── typelink │ │ ├── AliasVarBinds.hs │ │ ├── DataVarBinds.hs │ │ ├── ClassVarBinds.hs │ │ ├── LetSig.hs │ │ ├── InlineSig.hs │ │ ├── DataContext.hs │ │ ├── SimpleSig.hs │ │ ├── ForallBinds.hs │ │ ├── ContextBinds.hs │ │ └── ScopedTypeVar.hs ├── tests │ ├── BasicTest.hs │ ├── TypeLinkTest.hs │ └── Language │ │ └── Haskell │ │ └── Indexer │ │ └── Backend │ │ └── Ghc │ │ └── Test │ │ ├── TestShim.hs │ │ ├── TestHelper.hs │ │ └── TypeLinkTestBase.hs ├── src │ └── Language │ │ └── Haskell │ │ └── Indexer │ │ └── Backend │ │ ├── GhcEnv.hs │ │ ├── GhcArgs.hs │ │ └── GhcLens.hs └── haskell-indexer-backend-ghc.cabal ├── haskell-indexer-frontend-kythe ├── Setup.hs └── haskell-indexer-frontend-kythe.cabal ├── haskell-indexer-pipeline-ghckythe ├── Setup.hs ├── haskell-indexer-pipeline-ghckythe.cabal └── src │ └── Language │ └── Haskell │ └── Indexer │ └── Pipeline │ └── GhcKythe.hs ├── wrappers ├── stack-docker │ ├── everything │ │ └── Setup.hs │ ├── test-package │ │ ├── Setup.hs │ │ ├── app │ │ │ └── Main.hs │ │ ├── src │ │ │ └── Lib.hs │ │ └── test-package.cabal │ ├── Dockerfile │ ├── fake-docker │ │ └── ghc │ └── stack-lts-9.2.yaml └── stack │ └── ghc ├── haskell-indexer-pipeline-ghckythe-wrapper ├── Setup.hs ├── main │ └── GhcKytheWrapper.hs ├── haskell-indexer-pipeline-ghckythe-wrapper.cabal └── src │ └── Language │ └── Haskell │ └── Indexer │ └── Args.hs ├── kythe-proto ├── Setup.hs ├── kythe-proto.cabal ├── third_party │ └── kythe │ │ └── kythe │ │ └── proto │ │ └── storage.proto └── LICENSE ├── .gitignore ├── stack-example ├── stack.yaml ├── indexer-snapshot.yaml ├── gen-ghc-wrapper.sh └── README.md ├── kythe-verification ├── testdata │ └── basic │ │ ├── LocalRef.hs │ │ ├── CrossRef1.hs │ │ ├── TypeDef.hs │ │ ├── ModuleDocUri.hs │ │ ├── TypeOperators.hs │ │ ├── CrossRef2.hs │ │ ├── DocUri.hs │ │ ├── FunctionArgRef.hs │ │ ├── TypeVarInSig.hs │ │ ├── AssocType.hs │ │ ├── DataFam.hs │ │ ├── PatSyn.hs │ │ ├── Module.hs │ │ ├── ClosedFam.hs │ │ ├── ImpExpDefs.hs │ │ ├── ImportRefs.hs │ │ ├── Anchors.hs │ │ ├── RecursiveRef.hs │ │ ├── RecordWriteRef.hs │ │ ├── TypeclassRef.hs │ │ ├── TypeFam.hs │ │ ├── DataRef.hs │ │ └── RecordReadRef.hs ├── README.md └── test.sh ├── run-ghc-tests.sh ├── Dockerfile ├── nix ├── haskell-indexer-backend-core.nix ├── haskell-indexer-pathutil.nix ├── haskell-indexer-translate.nix ├── kythe-proto.nix ├── kythe-schema.nix ├── haskell-indexer-frontend-kythe.nix ├── text-offset.nix ├── haskell-indexer-pipeline-ghckythe.nix ├── haskell-indexer-pipeline-ghckythe-wrapper.nix ├── haskell-indexer-backend-ghc.nix └── default.nix ├── .hlint.yaml ├── .github └── workflows │ ├── analyze.yaml │ └── build.yaml ├── stack-ghc865.yaml ├── serve.sh ├── CONTRIBUTING.md ├── README.md ├── ROADMAP.md └── stack-ghc-8.10.yaml /stack.yaml: -------------------------------------------------------------------------------- 1 | stack-ghc865.yaml -------------------------------------------------------------------------------- /cabal.project: -------------------------------------------------------------------------------- 1 | packages: */*.cabal 2 | -------------------------------------------------------------------------------- /kythe-schema/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /text-offset/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-pathutil/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-plugin/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-translate/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-backend-core/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-frontend-kythe/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /wrappers/stack-docker/everything/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /wrappers/stack-docker/test-package/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ffi.c: -------------------------------------------------------------------------------- 1 | int pureFfiDep(int a) { 2 | return a * 2; 3 | } 4 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ExecutableMain.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | main = print 42 4 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe-wrapper/Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/AliasVarBinds.hs: -------------------------------------------------------------------------------- 1 | module AliasVarBinds where 2 | 3 | type T a = [a] 4 | -------------------------------------------------------------------------------- /kythe-proto/Setup.hs: -------------------------------------------------------------------------------- 1 | import Data.ProtoLens.Setup 2 | 3 | main = defaultMainGeneratingProtos "third_party/kythe" 4 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/UsedByTH.hs: -------------------------------------------------------------------------------- 1 | module UsedByTH where 2 | 3 | pureDep :: Int -> Int 4 | pureDep x = x*x 5 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/DataVarBinds.hs: -------------------------------------------------------------------------------- 1 | module DataVarBinds where 2 | 3 | data Xy a = Xy { unXy :: a } 4 | -------------------------------------------------------------------------------- /wrappers/stack-docker/test-package/app/Main.hs: -------------------------------------------------------------------------------- 1 | module Main where 2 | 3 | import Lib 4 | 5 | main :: IO () 6 | main = someFunc 7 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ImportRefsHiding.hs: -------------------------------------------------------------------------------- 1 | module ImportRefsHiding where 2 | 3 | import ImpExpDefs hiding (bar) 4 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/DataConWrap.hs: -------------------------------------------------------------------------------- 1 | module DataConWrap where 2 | 3 | data D a = A { unA :: !a } 4 | 5 | make = A 5 6 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/Imports.hs: -------------------------------------------------------------------------------- 1 | module Imports where 2 | 3 | import Data.Int 4 | import Data.List (groupBy, sortBy) 5 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/ClassVarBinds.hs: -------------------------------------------------------------------------------- 1 | module ClassVarBinds where 2 | 3 | class C a where 4 | foo :: a -> free -> free 5 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/Utf8.hs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/haskell-indexer/HEAD/haskell-indexer-backend-ghc/testdata/basic/Utf8.hs -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/LetSig.hs: -------------------------------------------------------------------------------- 1 | module LetSig where 2 | 3 | foo = 4 | let x :: a -> a 5 | x = undefined 6 | in x 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.stack-work 2 | .stack/ 3 | **/tags 4 | **/TAGS 5 | *.swp 6 | .projectile 7 | **/dist 8 | *.yaml.lock 9 | dist-newstyle 10 | .ghc.* 11 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TypeVsTerm.hs: -------------------------------------------------------------------------------- 1 | module TypeVsTerm where 2 | 3 | data X = X Int -- same name 4 | 5 | toTerm :: X 6 | toTerm = X 42 7 | -------------------------------------------------------------------------------- /stack-example/stack.yaml: -------------------------------------------------------------------------------- 1 | resolver: indexer-snapshot.yaml 2 | packages: 3 | - location: http://hackage.haskell.org/package/tasty-1.1.0.4/tasty-1.1.0.4.tar.gz 4 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/DocUri.hs: -------------------------------------------------------------------------------- 1 | module DocUri where 2 | 3 | import Data.Maybe (catMaybes) 4 | 5 | f :: [Maybe a] -> [a] 6 | f = catMaybes 7 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/DummyInclude.hs: -------------------------------------------------------------------------------- 1 | -- A dummy include file for use by other tests. 2 | -- It has a few lines and a function. 3 | includedFun = 42 4 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/LocalRef.hs: -------------------------------------------------------------------------------- 1 | module LocalRef where 2 | 3 | -- - @go ref FunGo 4 | f = go 5 | -- - @go defines/binding FunGo 6 | where go = undefined 7 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/InlineSig.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ScopedTypeVariables #-} 2 | module InlineSig where 3 | 4 | foo :: forall a . a 5 | foo = undefined :: a 6 | -------------------------------------------------------------------------------- /haskell-indexer-plugin/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Revision history for haskell-indexer-plugin 2 | 3 | ## 0.1.0.0 -- YYYY-mm-dd 4 | 5 | * First version. Released on an unsuspecting world. 6 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/FunCall.hs: -------------------------------------------------------------------------------- 1 | module FunCall where 2 | 3 | f = id 4 | fCaller = f 1 5 | 6 | g = id 7 | gReferrer = g 8 | 9 | h = id 10 | hMixed = h h 11 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/DataContext.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DatatypeContexts #-} 2 | module DataContext where 3 | 4 | -- Deprecated, but still. 5 | data (Eq a) => D a = MkD a 6 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/CrossRef1.hs: -------------------------------------------------------------------------------- 1 | -- - @"CrossRef1" defines/binding Module 2 | module CrossRef1 (foo) where 3 | 4 | foo :: Int 5 | -- -@foo defines/binding Fun 6 | foo = 42 7 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/TypeDef.hs: -------------------------------------------------------------------------------- 1 | module TypeDef where 2 | 3 | -- - @add ref/doc FunAdd 4 | add :: Int -> Int -> Int 5 | -- - @add defines/binding FunAdd 6 | add x y = x + y 7 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ArgRef.hs: -------------------------------------------------------------------------------- 1 | module ArgRef where 2 | 3 | a x = x 4 | 5 | b x = x + 1 6 | 7 | c x y = if True then x else y 8 | 9 | -- TODO(robinpalotai): add more 10 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ForeignImport.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ForeignFunctionInterface #-} 2 | module ForeignImport where 3 | 4 | foreign import ccall "pureFfiDep" pureFfiDep :: Int -> Int 5 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TypeOperators.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TypeOperators #-} 2 | 3 | module TypeOperator where 4 | 5 | data a + b = Sum a b 6 | 7 | mkSum :: a -> b -> a + b 8 | mkSum x y = Sum x y 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/SimpleSig.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE RankNTypes #-} 2 | module SimpleSig where 3 | 4 | unscoped :: a 5 | unscoped = undefined 6 | 7 | scoped :: forall a . a 8 | scoped = undefined 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/CppInclude.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE CPP #-} 2 | module CppInclude where 3 | 4 | beforeInclude = 1 5 | 6 | #include "DummyInclude.hs" 7 | 8 | afterInclude = includedFun + beforeInclude 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TypeClassRef.hs: -------------------------------------------------------------------------------- 1 | module TypeClassRef where 2 | 3 | called x = x 4 | 5 | class Foo a where 6 | foo :: a -> Int 7 | 8 | instance Foo () where 9 | foo _ = called 1 10 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/ForallBinds.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE RankNTypes #-} 2 | module ForallBinds where 3 | 4 | implicit :: a -> a 5 | implicit = undefined 6 | 7 | explicit :: forall a . a -> a 8 | explicit = undefined 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ForeignExport.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ForeignFunctionInterface #-} 2 | module ForeignExport where 3 | 4 | foreign export ccall exportedFun :: Int -> Int 5 | 6 | exportedFun :: Int -> Int 7 | exportedFun = (1+) 8 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/ModuleDocUri.hs: -------------------------------------------------------------------------------- 1 | module ModuleDocUri where 2 | 3 | -- - DataMaybeMod.doc/uri "https://hackage.haskell.org/package/base-4.12.0.0/docs/src/Data.Maybe.html" 4 | 5 | -- - @"Data.Maybe" ref/imports DataMaybeMod 6 | import Data.Maybe (mapMaybe) 7 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/TypeOperators.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TypeOperators #-} 2 | 3 | module TypeOperator where 4 | 5 | -- - @"+" defines/binding SumTypeOp 6 | data a + b = Sum a b 7 | 8 | -- - @"+" ref SumTypeOp 9 | mkSum :: a -> b -> a + b 10 | mkSum x y = Sum x y 11 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/DataDecl.hs: -------------------------------------------------------------------------------- 1 | module DataDecl where 2 | 3 | data AB = A | B Int 4 | 5 | f :: AB -> Int 6 | f A = 0 7 | f (B x) = x 8 | 9 | data Record = Record { recInt :: Int } 10 | 11 | g :: Record -> Int 12 | g (Record x) = x 13 | 14 | h = recInt 15 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TemplateHaskellCodeExec.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TemplateHaskell #-} 2 | module TemplateHaskellCodeExec where 3 | 4 | import Language.Haskell.TH.Syntax (lift) 5 | import Language.Haskell.TH 6 | import UsedByTH (pureDep) 7 | 8 | foo = $([|$(lift . pureDep $ 3)|]) 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TemplateHaskellQuotation.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TemplateHaskell #-} 2 | module TemplateHaskellQuotation where 3 | 4 | import Language.Haskell.TH 5 | 6 | $(runQ [d| 7 | thFun a = 41 8 | thVar = 3 9 | |]) 10 | 11 | afterTH = thFun thVar 12 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/ContextBinds.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE GADTs, RankNTypes #-} 2 | module ContextBinds where 3 | 4 | foo :: Eq c => c 5 | foo = undefined 6 | 7 | bar :: (Eq a, a ~ b) => b 8 | bar = undefined 9 | 10 | baz :: forall a . (Eq a) => a 11 | baz = undefined 12 | -------------------------------------------------------------------------------- /run-ghc-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euxo pipefail 3 | 4 | for i in stack-ghc865.yaml 5 | do 6 | export STACK_YAML=$i 7 | stack test haskell-indexer-backend-ghc 8 | stack install 9 | pushd kythe-verification 10 | ./test.sh 11 | popd 12 | done 13 | banner Success! 14 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/PatSyn.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE PatternSynonyms #-} 2 | module PatSyn where 3 | 4 | pattern UniSingle a <- [a] 5 | 6 | uniAsPattern (UniSingle x) = x 7 | 8 | pattern BiSingle a = [a] 9 | 10 | biAsExpr x = BiSingle x 11 | 12 | biAsPattern (BiSingle x) = x 13 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TemplateHaskellCodeExecFFI.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TemplateHaskell #-} 2 | module TemplateHaskellCodeExecFFI where 3 | 4 | import Language.Haskell.TH.Syntax (lift) 5 | import Language.Haskell.TH 6 | import ForeignImport (pureFfiDep) 7 | 8 | foo = $([|$(lift . pureFfiDep $ 4)|]) 9 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/TypeClass.hs: -------------------------------------------------------------------------------- 1 | module TypeClass where 2 | 3 | class Foo a where 4 | foo :: a -> String 5 | bar :: a 6 | 7 | instance Foo Int where 8 | foo = show 9 | bar = 42 10 | 11 | f x = foo x 12 | 13 | g :: Int -> String 14 | g = foo 15 | 16 | h :: Int 17 | h = bar 18 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/CrossRef2.hs: -------------------------------------------------------------------------------- 1 | module CrossRef2 where 2 | 3 | -- - @"CrossRef1" ref/imports Module 4 | import CrossRef1 (foo) 5 | -- - @"Data.List" ref/imports vname("base:Data.List", "", "", "", "haskell") 6 | import Data.List (groupBy, sortBy) 7 | 8 | bar :: Int 9 | -- - @foo ref Fun 10 | bar = foo + 1 11 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ImpExpDefs.hs: -------------------------------------------------------------------------------- 1 | module ImpExpDefs 2 | ( FooBar (MkFooBar, fbFoo, fbBar), 3 | bar, 4 | foo, 5 | ) 6 | where 7 | 8 | foo :: Int 9 | foo = 42 10 | 11 | bar :: Double 12 | bar = 42.0 13 | 14 | data FooBar 15 | = MkFooBar 16 | { fbFoo :: Int, 17 | fbBar :: Double 18 | } 19 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/typelink/ScopedTypeVar.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE ScopedTypeVariables #-} 2 | module ScopedTypeVar where 3 | 4 | unscoped :: a -> a 5 | unscoped = bar 6 | where 7 | bar :: a -> a 8 | bar = id 9 | 10 | scoped :: forall a . a -> a 11 | scoped = bars 12 | where 13 | bars :: a -> a 14 | bars = id 15 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/ImportRefs.hs: -------------------------------------------------------------------------------- 1 | module ImportRefs where 2 | 3 | import ImpExpDefs (bar, foo) -- IEVar 4 | import ImpExpDefs (FooBar) -- IEThingAbs 5 | import ImpExpDefs (FooBar (..)) -- IEThingAll 6 | import ImpExpDefs (FooBar (MkFooBar, fbFoo, fbBar)) -- IEThingWith 7 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/DocUri.hs: -------------------------------------------------------------------------------- 1 | module DocUri where 2 | 3 | -- - IODecl.doc/uri "https://hackage.haskell.org/package/ghc-prim-0.5.3/docs/src/GHC.Types.html#IO" 4 | -- - PutStrLnDecl.doc/uri "https://hackage.haskell.org/package/base-4.12.0.0/docs/src/System.IO.html#putStrLn" 5 | 6 | -- - @IO ref IODecl 7 | f :: IO () 8 | -- - @putStrLn ref PutStrLnDecl 9 | f = putStrLn "hello" 10 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/RecursiveRef.hs: -------------------------------------------------------------------------------- 1 | module RecursiveRef where 2 | {-# ANN module "HLint: ignore Eta reduce" #-} 3 | 4 | recNoSig x = recNoSig x 5 | 6 | dummy = localRecNoSig 7 | where 8 | localRecNoSig x = localRecNoSig x 9 | 10 | recWithSig :: a -> a 11 | recWithSig x = recWithSig x 12 | 13 | mutualNoSigA = mutualNoSigB 14 | mutualNoSigB = mutualNoSigA 15 | 16 | etaNoSig = etaNoSig 17 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/FunctionArgRef.hs: -------------------------------------------------------------------------------- 1 | module FunctionArgRef where 2 | 3 | -- Test simple argument reference. 4 | -- - @a defines/binding ParamFA 5 | f a = 6 | -- - @a ref ParamFA 7 | a 8 | 9 | -- Test pattern-matched argument reference. 10 | -- - @a defines/binding ParamGA 11 | -- - @b defines/binding ParamGB 12 | g (a,b) = 13 | -- - @a ref ParamGA 14 | -- - @b ref ParamGB 15 | a + b 16 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/TypeVarInSig.hs: -------------------------------------------------------------------------------- 1 | -- This file is just for checking that Kythe verifier doesn't fail with 2 | -- duplicate anchors when type variables are used in type signatures. 3 | module TypeVarInSig where 4 | 5 | -- The 'a' below causes duplicate anchors, and makes kythe-verifier fail 6 | -- without the `--ignore_dups` flag. 7 | isJust :: Maybe a -> Bool 8 | isJust (Just x) = True 9 | isJust Nothing = False 10 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/AssocType.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DataKinds #-} 2 | {-# LANGUAGE KindSignatures #-} 3 | {-# LANGUAGE TypeFamilies #-} 4 | 5 | module AssocType where 6 | 7 | import Data.Kind (Type) 8 | 9 | class Bar a where 10 | -- - @Baz defines/binding FamBaz 11 | type Baz a :: Type 12 | 13 | instance Bar Int where 14 | -- - @Baz ref FamBaz 15 | type Baz Int = Bool 16 | 17 | -- - @Baz ref FamBaz 18 | type B'azInt = Baz Int 19 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/DataFam.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DataKinds #-} 2 | {-# LANGUAGE KindSignatures #-} 3 | {-# LANGUAGE TypeFamilies #-} 4 | 5 | module DataFam where 6 | 7 | import Data.Kind (Type) 8 | import GHC.TypeNats (Nat) 9 | 10 | -- - @FooData defines/binding DataFamFoo 11 | data family FooData (a :: Nat) :: Type 12 | 13 | -- - @FooData ref DataFamFoo 14 | data instance FooData 4 = F'ooData4 Int 15 | 16 | -- - @FooData ref DataFamFoo 17 | type F'ooData4 = FooData 4 18 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/PatSyn.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE PatternSynonyms #-} 2 | module PatSyn where 3 | 4 | -- - @UniSingle defines/binding PatUniSingle 5 | pattern UniSingle a <- [a] 6 | 7 | -- - @UniSingle ref PatUniSingle 8 | uniAsPattern (UniSingle x) = x 9 | 10 | -- - @BiSingle defines/binding PatBiSingle 11 | pattern BiSingle a = [a] 12 | 13 | -- - @BiSingle ref PatBiSingle 14 | biAsExpr x = BiSingle x 15 | 16 | -- - @BiSingle ref PatBiSingle 17 | biAsPattern (BiSingle x) = x 18 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/Module.hs: -------------------------------------------------------------------------------- 1 | -- Haskell package+module combination corresponds to Kythe package. 2 | -- - @Module defines/binding Pkg 3 | -- - Pkg.node/kind package 4 | module Module where 5 | 6 | -- - @someTopLevel defines/binding TopDecl 7 | -- - TopDecl childof Pkg 8 | someTopLevel = 3 9 | 10 | -- Files belonging to a package+module are children of that package+module. 11 | -- - File = vname("", _, _, "testdata/basic/Module.hs", "").node/kind file 12 | -- - File childof Pkg 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the following command to build image: 2 | # 3 | # docker build -t hsidx-build . 4 | FROM fpco/stack-build:lts-9.2 5 | 6 | # chmod needed if set-user is true in stackage.yaml's docker section. 7 | RUN cd /tmp && \ 8 | wget https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip && \ 9 | unzip protoc-3.3.0-linux-x86_64.zip bin/protoc -d /usr/local && \ 10 | rm protoc-3.3.0-linux-x86_64.zip && \ 11 | chmod 755 /usr/local/bin/protoc 12 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/RecordRead.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NamedFieldPuns, RecordWildCards #-} 2 | module RecordRead where 3 | 4 | data Rec = Rec { field :: Int, baar :: Int } 5 | 6 | unpack (Rec f b) = f 7 | 8 | access r = field r 9 | 10 | wildcard Rec{..} = field 11 | 12 | punned Rec{field} = field 13 | 14 | reassigned Rec{field=bla} = bla 15 | 16 | data Complex = Complex { comp :: (Int,Int) } 17 | complexMatched Complex{comp=(a,b)} = a 18 | 19 | rightWild Rec{baar,..} = field + baar 20 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/LocalRef.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE BangPatterns #-} 2 | module LocalRef where 3 | import Prelude hiding (lookup) 4 | 5 | -- 6 | difference a b = foldlWithKey' go 7 | where 8 | go = case "oaeu" of 9 | "oeu" -> "oqeju" 10 | _ -> "oeuoeu" 11 | {-# INLINABLE difference #-} 12 | 13 | -- 14 | intersection a b = foldlWithKey' go (5 :: Int) 3 1 15 | where 16 | go x y z = x + y + z 17 | {-# INLINABLE intersection #-} 18 | 19 | foldlWithKey' = id 20 | -------------------------------------------------------------------------------- /nix/haskell-indexer-backend-core.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, stdenv, text 2 | , indexerSrc }: 3 | mkDerivation { 4 | pname = "haskell-indexer-backend-core"; 5 | version = "0.1.0.0"; 6 | src = indexerSrc; 7 | postUnpack = "sourceRoot+=/haskell-indexer-backend-core; echo source root reset to $sourceRoot"; 8 | libraryHaskellDepends = [ base text ]; 9 | homepage = "https://github.com/google/haskell-indexer"; 10 | description = "Code common to all indexer backends"; 11 | license = stdenv.lib.licenses.asl20; 12 | } 13 | -------------------------------------------------------------------------------- /nix/haskell-indexer-pathutil.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, filepath, stdenv, text 2 | , indexerSrc}: 3 | mkDerivation { 4 | pname = "haskell-indexer-pathutil"; 5 | version = "0.1.0.0"; 6 | src = indexerSrc; 7 | postUnpack = "sourceRoot+=/haskell-indexer-pathutil; echo source root reset to $sourceRoot"; 8 | libraryHaskellDepends = [ base filepath text ]; 9 | homepage = "https://github.com/google/haskell-indexer"; 10 | description = "Utilities for dealing with filepaths"; 11 | license = stdenv.lib.licenses.asl20; 12 | } 13 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/ClosedFam.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DataKinds #-} 2 | {-# LANGUAGE KindSignatures #-} 3 | {-# LANGUAGE TypeFamilies #-} 4 | 5 | module TypeFam where 6 | 7 | import Data.Kind (Type) 8 | import GHC.TypeNats (Nat) 9 | 10 | -- - @FooClosed defines/binding ClosedFamFoo 11 | type family FooClosed (a :: Nat) where 12 | -- - @FooClosed ref ClosedFamFoo 13 | FooClosed 4 = Int 14 | -- - @FooClosed ref ClosedFamFoo 15 | FooClosed a = Bool 16 | 17 | -- - @FooClosed ref ClosedFamFoo 18 | type F'ooClosed4 = FooClosed 4 19 | -------------------------------------------------------------------------------- /nix/haskell-indexer-translate.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, filepath, stdenv, text 2 | , indexerSrc }: 3 | mkDerivation { 4 | pname = "haskell-indexer-translate"; 5 | version = "0.1.0.0"; 6 | src = indexerSrc; 7 | postUnpack = "sourceRoot+=/haskell-indexer-translate; echo source root reset to $sourceRoot"; 8 | libraryHaskellDepends = [ base bytestring filepath text ]; 9 | homepage = "https://github.com/google/haskell-indexer"; 10 | description = "Translation layer isolating compiler backends from frontends"; 11 | license = stdenv.lib.licenses.asl20; 12 | } 13 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/ImpExpDefs.hs: -------------------------------------------------------------------------------- 1 | module ImpExpDefs 2 | -- - @FooBar ref TypeD 3 | -- - @MkFB ref CtorD 4 | -- - @fbFoo ref FieldFbFoo 5 | ( FooBar (MkFB, fbFoo), 6 | -- - @bar ref BarVar 7 | bar, 8 | -- - @foo ref FooVar 9 | foo, 10 | ) 11 | where 12 | 13 | foo :: Int 14 | -- - @foo defines/binding FooVar 15 | foo = 42 16 | 17 | bar :: Double 18 | -- - @bar defines/binding BarVar 19 | bar = 42.0 20 | 21 | -- - @FooBar defines/binding TypeD 22 | -- - @MkFB defines/binding CtorD 23 | -- - @fbFoo defines/binding FieldFbFoo 24 | data FooBar = MkFB {fbFoo :: Int} 25 | -------------------------------------------------------------------------------- /nix/kythe-proto.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, Cabal, proto-lens-runtime 2 | , proto-lens-setup, stdenv 3 | , indexerSrc }: 4 | mkDerivation { 5 | pname = "kythe-proto"; 6 | version = "0.1.0.0"; 7 | src = indexerSrc; 8 | postUnpack = "sourceRoot+=/kythe-proto; echo source root reset to $sourceRoot"; 9 | setupHaskellDepends = [ base Cabal proto-lens-setup ]; 10 | libraryHaskellDepends = [ base proto-lens-runtime ]; 11 | homepage = "https://github.com/google/haskell-indexer"; 12 | description = "Proto bindings for Kythe protobufs"; 13 | license = stdenv.lib.licenses.asl20; 14 | } 15 | -------------------------------------------------------------------------------- /stack-example/indexer-snapshot.yaml: -------------------------------------------------------------------------------- 1 | resolver: nightly-2018-10-15 # GHC 8.6.1 2 | name: haskell-indexer-snapshot 3 | 4 | packages: 5 | - git: https://github.com/google/haskell-indexer 6 | commit: origin/master 7 | subdirs: 8 | - haskell-indexer-backend-core 9 | - haskell-indexer-backend-ghc 10 | - haskell-indexer-frontend-kythe 11 | - haskell-indexer-pathutil 12 | - haskell-indexer-pipeline-ghckythe 13 | - haskell-indexer-pipeline-ghckythe-wrapper 14 | - haskell-indexer-plugin 15 | - haskell-indexer-translate 16 | - kythe-proto 17 | - kythe-schema 18 | - text-offset 19 | -------------------------------------------------------------------------------- /nix/kythe-schema.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, data-default 2 | , kythe-proto, lens-family, proto-lens, stdenv, text 3 | , indexerSrc }: 4 | mkDerivation { 5 | pname = "kythe-schema"; 6 | version = "0.1.0.0"; 7 | src = indexerSrc; 8 | postUnpack = "sourceRoot+=/kythe-schema; echo source root reset to $sourceRoot"; 9 | libraryHaskellDepends = [ 10 | base bytestring data-default kythe-proto lens-family proto-lens 11 | text 12 | ]; 13 | homepage = "https://github.com/google/haskell-indexer"; 14 | description = "Library for emitting Kythe schema entries"; 15 | license = stdenv.lib.licenses.asl20; 16 | } 17 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/ImportRefs.hs: -------------------------------------------------------------------------------- 1 | module ImportRefs where 2 | 3 | -- - @foo ref/imports FooVar 4 | -- - @bar ref/imports BarVar 5 | import ImpExpDefs (bar, foo) -- IEVar 6 | 7 | -- - @FooBar ref/imports TypeD 8 | import ImpExpDefs (FooBar) -- IEThingAbs 9 | 10 | -- - @FooBar ref/imports TypeD 11 | import ImpExpDefs (FooBar (..)) -- IEThingAll 12 | 13 | -- - @FooBar ref/imports TypeD 14 | -- - @MkFB ref/imports CtorD 15 | -- - @fbFoo ref/imports FieldFbFoo 16 | import ImpExpDefs (FooBar (MkFB, fbFoo)) -- IEThingWith 17 | 18 | -- - @bar ref BarVar 19 | import ImpExpDefs hiding (bar) 20 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/Anchors.hs: -------------------------------------------------------------------------------- 1 | module Anchors where 2 | 3 | -- Anchors are childof the files they are defined in. 4 | -- - @f defines/binding FunF 5 | f = 6 | -- Kythe usually assign source anchors of ref/calls to their parent context. 7 | -- - @callMeMaybe childof FunF 8 | callMeMaybe undefined 9 | 10 | -- - @g defines/binding FunG 11 | -- - @x childof FunG 12 | g = x 13 | -- Even calls of local bindings are put under the top-level scope, since 14 | -- fine-grained scoping is not a goal for Kythe. 15 | -- - @callMeMaybe childof FunG 16 | where x = callMeMaybe undefined 17 | 18 | callMeMaybe :: Int -> Int 19 | callMeMaybe = undefined 20 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/testdata/basic/RecordWrite.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NamedFieldPuns, RecordWildCards #-} 2 | module RecordWrite where 3 | 4 | data Rec = Rec 5 | { foo :: Int 6 | , bar :: Int 7 | } 8 | 9 | create = Rec 1 2 10 | 11 | createRec = let x = 1 in Rec { bar = 2, foo = x } 12 | 13 | wildcard = 14 | let foo = 1 15 | bar = 3 16 | in Rec{..} 17 | 18 | pun = 19 | let foo = 1 20 | bar = 2 21 | in Rec{foo,..} 22 | 23 | update r = r { bar = 1 } 24 | 25 | unpackCreate Rec{..} = Rec 26 | { foo = 1 27 | , bar = bar 28 | } 29 | 30 | data Multi = A { m :: Int } | B { m :: Int } 31 | 32 | multiUpdate mu = mu { m = 5 } 33 | -------------------------------------------------------------------------------- /wrappers/stack-docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build with 2 | # docker build -t fake-build \ 3 | # --build-arg wrapper_path=path/to/bin/ghc_kythe_wrapper . 4 | FROM fpco/stack-build:lts-9.2 5 | 6 | ARG wrapper_path 7 | COPY $wrapper_path /usr/local/bin/ghc_kythe_wrapper 8 | 9 | # The original is just a symlink pointing to ghc-8.0.2. 10 | # The fake ghc will call the latter explicitly. 11 | COPY fake-docker/ghc /usr/local/bin/fhc 12 | 13 | # chmod needed if set-user is true in stackage.yaml's docker section. 14 | RUN mv /opt/ghc/8.0.2/bin/ghc /opt/ghc/8.0.2/bin/ohc && \ 15 | ln -s /usr/local/bin/fhc /opt/ghc/8.0.2/bin/ghc && \ 16 | chmod 755 /usr/local/bin/fhc /usr/local/bin/ghc_kythe_wrapper 17 | -------------------------------------------------------------------------------- /nix/haskell-indexer-frontend-kythe.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, conduit 2 | , haskell-indexer-translate, kythe-schema, mmorph, mtl, stdenv 3 | , text, text-offset, transformers 4 | , indexerSrc }: 5 | mkDerivation { 6 | pname = "haskell-indexer-frontend-kythe"; 7 | version = "0.1.0.0"; 8 | src = indexerSrc; 9 | postUnpack = "sourceRoot+=/haskell-indexer-frontend-kythe; echo source root reset to $sourceRoot"; 10 | libraryHaskellDepends = [ 11 | base bytestring conduit haskell-indexer-translate kythe-schema 12 | mmorph mtl text text-offset transformers 13 | ]; 14 | homepage = "https://github.com/google/haskell-indexer"; 15 | description = "Emits Kythe schema based on translation layer data"; 16 | license = stdenv.lib.licenses.asl20; 17 | } 18 | -------------------------------------------------------------------------------- /nix/text-offset.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, HUnit, QuickCheck 2 | , stdenv, test-framework, test-framework-hunit 3 | , test-framework-quickcheck2, text, vector 4 | , indexerSrc }: 5 | mkDerivation { 6 | pname = "text-offset"; 7 | version = "0.1.0.0"; 8 | src = indexerSrc; 9 | postUnpack = "sourceRoot+=/text-offset; echo source root reset to $sourceRoot"; 10 | libraryHaskellDepends = [ base text vector ]; 11 | testHaskellDepends = [ 12 | base bytestring HUnit QuickCheck test-framework 13 | test-framework-hunit test-framework-quickcheck2 text 14 | ]; 15 | homepage = "https://github.com/google/haskell-indexer"; 16 | description = "Library for converting between line/column and byte offset"; 17 | license = stdenv.lib.licenses.asl20; 18 | } 19 | -------------------------------------------------------------------------------- /wrappers/stack-docker/test-package/src/Lib.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | module Lib 16 | ( someFunc 17 | ) where 18 | 19 | someFunc :: IO () 20 | someFunc = putStrLn "someFunc" 21 | -------------------------------------------------------------------------------- /wrappers/stack-docker/fake-docker/ghc: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Don't forget to mount /logs of the guest to some host dir. 3 | OUT=/logs 4 | PKG=$(basename "$(pwd)") 5 | 6 | log() { 7 | echo "$1" >> $OUT/$PKG.log 8 | } 9 | 10 | log "========= FAKE GHC =======" 11 | log " == pwd: $(pwd)" 12 | for arg 13 | do 14 | if [ "$arg" = "-static" ]; then 15 | STATIC="true" 16 | fi 17 | log "$arg" 18 | done 19 | REALGHC=/opt/ghc/8.0.2/bin/ohc 20 | log "== Passing through.." 21 | log "$REALGHC $@" 22 | $REALGHC "$@" 23 | RESULT=$? 24 | if [ "$STATIC" = "true" ]; then 25 | log "== Invoking indexer" 26 | ghc_kythe_wrapper --drop_path_prefix './' --prepend_path_prefix "$PKG/" -- \ 27 | "$@" >> $OUT/$PKG.entries 2> $OUT/$PKG.stderr 28 | [ "$?" != "0" ] && echo "$PKG had error" >> $OUT/errors 29 | fi 30 | exit $RESULT 31 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/tests/BasicTest.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | module Main (main) where 16 | 17 | import Language.Haskell.Indexer.Backend.Ghc.Test.TestShim (runTests) 18 | import Language.Haskell.Indexer.Backend.Ghc.Test.BasicTestBase (allTests) 19 | 20 | main :: IO () 21 | main = runTests allTests 22 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/tests/TypeLinkTest.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | module Main (main) where 16 | 17 | import Language.Haskell.Indexer.Backend.Ghc.Test.TestShim (runTests) 18 | import Language.Haskell.Indexer.Backend.Ghc.Test.TypeLinkTestBase (allTests) 19 | 20 | main :: IO () 21 | main = runTests allTests 22 | -------------------------------------------------------------------------------- /nix/haskell-indexer-pipeline-ghckythe.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, conduit, filepath 2 | , haskell-indexer-backend-core, haskell-indexer-backend-ghc 3 | , haskell-indexer-frontend-kythe, haskell-indexer-translate 4 | , kythe-schema, mmorph, mtl, stdenv, text 5 | , indexerSrc }: 6 | mkDerivation { 7 | pname = "haskell-indexer-pipeline-ghckythe"; 8 | version = "0.1.0.0"; 9 | src = indexerSrc; 10 | postUnpack = "sourceRoot+=/haskell-indexer-pipeline-ghckythe; echo source root reset to $sourceRoot"; 11 | libraryHaskellDepends = [ 12 | base bytestring conduit filepath haskell-indexer-backend-core 13 | haskell-indexer-backend-ghc haskell-indexer-frontend-kythe 14 | haskell-indexer-translate kythe-schema mmorph mtl text 15 | ]; 16 | homepage = "https://github.com/google/haskell-indexer"; 17 | description = "Gets GHC invocation arguments and streams Kythe entries"; 18 | license = stdenv.lib.licenses.asl20; 19 | } 20 | -------------------------------------------------------------------------------- /wrappers/stack-docker/stack-lts-9.2.yaml: -------------------------------------------------------------------------------- 1 | resolver: lts-9.2 2 | 3 | packages: 4 | - test-package 5 | - everything 6 | 7 | ghc-options: 8 | # https://ghc.haskell.org/trac/ghc/ticket/12147 9 | # https://github.com/gibiansky/IHaskell/issues/636 10 | bindings-GLFW: -opta-Wa,-mrelax-relocations=no 11 | cipher-aes: -opta-Wa,-mrelax-relocations=no 12 | skein: -opta-Wa,-mrelax-relocations=no 13 | atomic-primops: -opta-Wa,-mrelax-relocations=no 14 | bcrypt: -opta-Wa,-mrelax-relocations=no 15 | cryptonite: -opta-Wa,-mrelax-relocations=no 16 | direct-sqlite: -opta-Wa,-mrelax-relocations=no 17 | double-conversion: -opta-Wa,-mrelax-relocations=no 18 | glib: -opta-Wa,-mrelax-relocations=no 19 | HsOpenSSL: -opta-Wa,-mrelax-relocations=no 20 | inline-r: -opta-Wa,-mrelax-relocations=no 21 | yaml: -opta-Wa,-mrelax-relocations=no 22 | 23 | docker: 24 | enable: false # Use stack --docker instead 25 | image: fake-build 26 | mount: 27 | - "/tmp/kythe-entries:/logs" 28 | -------------------------------------------------------------------------------- /.hlint.yaml: -------------------------------------------------------------------------------- 1 | # HLint configuration file 2 | # https://github.com/ndmitchell/hlint 3 | ########################## 4 | 5 | # Warnings currently triggered by the code. 6 | # These may or may not be code we want to fix. 7 | - ignore: {name: "Eta reduce"} # 7 hints 8 | - ignore: {name: "Functor law"} # 1 hint 9 | - ignore: {name: "Fuse concatMap/map"} # 1 hint 10 | - ignore: {name: "Redundant $"} # 1 hint 11 | - ignore: {name: "Redundant $!"} # 8 hints 12 | - ignore: {name: "Redundant bracket"} # 6 hints 13 | - ignore: {name: "Redundant flip"} # 2 hints 14 | - ignore: {name: "Redundant fmap"} # 2 hints 15 | - ignore: {name: "Replace case with fromMaybe"} # 1 hint 16 | - ignore: {name: "Unused LANGUAGE pragma"} # 12 hints 17 | - ignore: {name: "Use ++"} # 5 hints 18 | - ignore: {name: "Use <&>"} # 1 hint 19 | - ignore: {name: "Use fewer imports"} # 2 hints 20 | - ignore: {name: "Use maybe"} # 5 hints 21 | - ignore: {name: "Use newtype instead of data"} # 15 hints 22 | - ignore: {name: "Use section"} # 1 hint 23 | - ignore: {name: "Use traverse_"} # 1 hint 24 | -------------------------------------------------------------------------------- /haskell-indexer-pathutil/haskell-indexer-pathutil.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-pathutil 2 | version: 0.1.0.0 3 | synopsis: Utilities for dealing with filepaths. 4 | description: Part of haskell-indexer, see top-level README.md. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Haskell.Indexer.Util.Path 18 | build-depends: base >=4.8 && <5 19 | , filepath 20 | , text 21 | ghc-options: -Wall 22 | -Wcompat 23 | -Wincomplete-record-updates 24 | -Wincomplete-uni-patterns 25 | -Wredundant-constraints 26 | default-language: Haskell2010 27 | -------------------------------------------------------------------------------- /haskell-indexer-backend-core/haskell-indexer-backend-core.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-backend-core 2 | version: 0.1.0.0 3 | synopsis: Code common to all indexer backends. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Haskell.Indexer.Backend.AnalysisOptions 18 | build-depends: base >=4.8 && <5 19 | , text 20 | ghc-options: -Wall 21 | -Wcompat 22 | -Wincomplete-record-updates 23 | -Wincomplete-uni-patterns 24 | -Wredundant-constraints 25 | default-language: Haskell2010 26 | -------------------------------------------------------------------------------- /nix/haskell-indexer-pipeline-ghckythe-wrapper.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, ghc 2 | , haskell-indexer-backend-core, haskell-indexer-backend-ghc 3 | , haskell-indexer-pathutil, haskell-indexer-pipeline-ghckythe 4 | , haskell-indexer-translate, kythe-schema, optparse-applicative 5 | , proto-lens, stdenv, text 6 | , indexerSrc }: 7 | mkDerivation { 8 | pname = "haskell-indexer-pipeline-ghckythe-wrapper"; 9 | version = "0.1.0.0"; 10 | src = indexerSrc; 11 | postUnpack = "sourceRoot+=/haskell-indexer-pipeline-ghckythe-wrapper; echo source root reset to $sourceRoot"; 12 | isLibrary = true; 13 | isExecutable = true; 14 | libraryHaskellDepends = [ 15 | base bytestring ghc haskell-indexer-backend-core 16 | haskell-indexer-backend-ghc haskell-indexer-pathutil 17 | haskell-indexer-pipeline-ghckythe haskell-indexer-translate 18 | kythe-schema optparse-applicative proto-lens text 19 | ]; 20 | executableHaskellDepends = [ base ]; 21 | homepage = "https://github.com/google/haskell-indexer"; 22 | description = "Executable able to take GHC arguments and emitting Kythe entries"; 23 | license = stdenv.lib.licenses.asl20; 24 | } 25 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/RecursiveRef.hs: -------------------------------------------------------------------------------- 1 | module RecursiveRef where 2 | {-# ANN module "HLint: ignore Eta reduce" #-} 3 | 4 | -- Recursive function call without type signature targets the monomorphic 5 | -- binding. This verifies that we handle the case. 6 | -- - @recNoSig defines/binding FunRNS 7 | recNoSig x = 8 | -- - @recNoSig ref FunRNS 9 | recNoSig x 10 | 11 | -- - @localRecNoSig ref FunLRNS 12 | dummy = localRecNoSig 13 | where 14 | -- - @localRecNoSig defines/binding FunLRNS 15 | localRecNoSig x = 16 | -- - @localRecNoSig ref FunLRNS 17 | localRecNoSig x 18 | 19 | -- Recursive call to function with type signature targets the polymorphic 20 | -- binding. 21 | recWithSig :: Int -> Int 22 | -- - @recWithSig defines/binding FunRWS 23 | recWithSig x = 24 | -- - @recWithSig ref FunRWS 25 | recWithSig x 26 | 27 | -- - @mutualNoSigA defines/binding FunMA 28 | -- - @mutualNoSigB ref FunMB 29 | mutualNoSigA = mutualNoSigB 30 | -- - @mutualNoSigB defines/binding FunMB 31 | -- - @mutualNoSigA ref FunMA 32 | mutualNoSigB = mutualNoSigA 33 | 34 | -- - @etaNoSig defines/binding FunENS 35 | etaNoSig = 36 | -- - @etaNoSig ref FunENS 37 | etaNoSig 38 | -------------------------------------------------------------------------------- /.github/workflows/analyze.yaml: -------------------------------------------------------------------------------- 1 | name: Analyze 2 | permissions: read-all 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | 8 | pull_request: 9 | paths: 10 | - '**.hs' 11 | - '.hlint.yaml' 12 | - '.github/workflows/hlint.yaml' 13 | 14 | jobs: 15 | hlint: 16 | runs-on: "ubuntu-latest" 17 | permissions: 18 | # Needed to upload results to GitHub code scanning. 19 | security-events: write 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 24 | 25 | - name: HLint 26 | uses: haskell-actions/hlint-scan@3e2feb228d5d90db9ba2bb93c720d83e4c06681c # v1.2.0 27 | with: 28 | path: | 29 | haskell-indexer-backend-core 30 | haskell-indexer-backend-ghc/src 31 | haskell-indexer-backend-ghc/tests 32 | haskell-indexer-frontend-kythe 33 | haskell-indexer-pathutil 34 | haskell-indexer-pipeline-ghckythe 35 | haskell-indexer-pipeline-ghckythe-wrapper 36 | haskell-indexer-plugin 37 | haskell-indexer-translate 38 | kythe-schema 39 | text-offset 40 | -------------------------------------------------------------------------------- /stack-ghc865.yaml: -------------------------------------------------------------------------------- 1 | resolver: lts-14.7 2 | 3 | packages: 4 | - haskell-indexer-backend-core 5 | - haskell-indexer-backend-ghc 6 | - haskell-indexer-frontend-kythe 7 | - haskell-indexer-pathutil 8 | - haskell-indexer-pipeline-ghckythe 9 | - haskell-indexer-pipeline-ghckythe-wrapper 10 | - haskell-indexer-translate 11 | - kythe-proto 12 | - kythe-schema 13 | - text-offset 14 | 15 | extra-deps: 16 | - proto-lens-0.7.0.0@sha256:2c44a62375f7712f9381f84b1d30cee2f94384f1c98801db2f4450359a8e5036,3036 17 | - proto-lens-protoc-0.7.0.0@sha256:04e57ec59ce21dc781b9debe063f03d85a7ec2ad1ee5c2deb1c30282119e9119,2304 18 | - proto-lens-runtime-0.7.0.0@sha256:d6cfab159a63f5c42a1a507638c674a714dfa5f69a06f559e8da840eaafde3ab,3115 19 | - proto-lens-setup-0.4.0.4@sha256:c44d1b6d68a8faf1d2f9c7ca492ce3c9c2ee66d255452c683f762f10c9ebe430,3185 20 | - ghc-source-gen-0.4.0.0@sha256:b6cdde6b183e70f6fbed3b0a45ad6640204b636e9a7fb898c5d7ec79719f14a1,3698 21 | 22 | # Allow our custom Setup.hs scripts to import Data.ProtoLens.Setup from the version of 23 | # `proto-lens-protoc` in stack's local DB. See: 24 | # https://github.com/google/proto-lens/blob/master/README.md#using-cabal 25 | explicit-setup-deps: 26 | "*": true 27 | 28 | nix: 29 | packages: [gcc, protobuf] 30 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/tests/Language/Haskell/Indexer/Backend/Ghc/Test/TestShim.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | -- | Shim to easily run Translate layer tests using standard GHC environment. 16 | module Language.Haskell.Indexer.Backend.Ghc.Test.TestShim (runTests) where 17 | 18 | import Test.Framework (defaultMain, Test) 19 | 20 | import Language.Haskell.Indexer.Backend.GhcArgs (defaultGhcArgs) 21 | import Language.Haskell.Indexer.Backend.Ghc.Test.TestHelper (TestEnv(..)) 22 | 23 | runTests :: (TestEnv -> [Test]) -> IO () 24 | runTests allTests = do 25 | let env = TestEnv "testdata" defaultGhcArgs 26 | defaultMain (allTests env) 27 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/RecordWriteRef.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NamedFieldPuns, RecordWildCards #-} 2 | module RecordWrite where 3 | 4 | data Rec = Rec 5 | -- - @foo defines/binding FieldFoo 6 | { foo :: Int 7 | -- - @bar defines/binding FieldBar 8 | , bar :: Int 9 | } 10 | 11 | create = Rec 1 2 12 | 13 | -- - @foo ref FieldFoo 14 | -- - @bar ref FieldBar 15 | createRec = let x = 1 in Rec { bar = 2, foo = x } 16 | 17 | wildcard = 18 | -- - @foo ref FieldFoo 19 | let foo = 1 20 | -- - @bar ref FieldBar 21 | bar = 3 22 | -- - @"Rec{..}" ref FieldFoo 23 | -- - @"Rec{..}" ref FieldBar 24 | in Rec{..} 25 | 26 | pun = 27 | let foo = 1 28 | bar = 2 29 | -- - @foo ref FieldFoo 30 | -- - @"Rec{foo,..}" ref FieldBar 31 | in Rec{foo,..} 32 | 33 | -- - @bar ref FieldBar 34 | update r = r { bar = 1 } 35 | 36 | unpackCreate Rec{..} = Rec 37 | -- - @foo ref FieldFoo 38 | { foo = 1 39 | -- - @bar ref FieldBar 40 | , bar = 41 | bar 42 | } 43 | 44 | data Multi 45 | -- - @m defines/binding FieldM 46 | = A { m :: Int } 47 | -- Note: no bindings / refs at @m. 48 | | B { m :: Int } 49 | 50 | multiUpdate mu = mu 51 | -- - @m ref FieldM 52 | { m = 5 } 53 | -------------------------------------------------------------------------------- /nix/haskell-indexer-backend-ghc.nix: -------------------------------------------------------------------------------- 1 | { mkDerivation, base, bytestring, Cabal, containers, directory 2 | , filepath, ghc, ghc-paths, hashable 3 | , haskell-indexer-backend-core, haskell-indexer-translate, HUnit 4 | , mtl, network-uri, reflection, semigroups, stdenv, temporary 5 | , test-framework, test-framework-hunit, text, text-offset 6 | , transformers, unix, unordered-containers 7 | , indexerSrc }: 8 | mkDerivation { 9 | pname = "haskell-indexer-backend-ghc"; 10 | version = "0.1.0.0"; 11 | src = indexerSrc; 12 | postUnpack = "sourceRoot+=/haskell-indexer-backend-ghc; echo source root reset to $sourceRoot"; 13 | libraryHaskellDepends = [ 14 | base Cabal containers directory filepath ghc ghc-paths hashable 15 | haskell-indexer-backend-core haskell-indexer-translate mtl 16 | network-uri reflection text transformers unix unordered-containers 17 | ]; 18 | testHaskellDepends = [ 19 | base bytestring directory filepath ghc haskell-indexer-backend-core 20 | haskell-indexer-translate HUnit semigroups temporary test-framework 21 | test-framework-hunit text text-offset transformers 22 | ]; 23 | homepage = "https://github.com/google/haskell-indexer"; 24 | description = "Indexing code using GHC API"; 25 | license = stdenv.lib.licenses.asl20; 26 | } 27 | -------------------------------------------------------------------------------- /wrappers/stack-docker/test-package/test-package.cabal: -------------------------------------------------------------------------------- 1 | name: test-package 2 | version: 0.1.0.0 3 | license: Apache-2.0 4 | license-file: LICENSE 5 | author: Robin Palotai 6 | maintainer: robinpalotai@google.com 7 | copyright: Google Inc. 8 | category: Web 9 | build-type: Simple 10 | cabal-version: >=1.10 11 | 12 | library 13 | hs-source-dirs: src 14 | exposed-modules: Lib 15 | build-depends: base >= 4.7 && < 5 16 | ghc-options: -Wall 17 | -Wcompat 18 | -Wincomplete-record-updates 19 | -Wincomplete-uni-patterns 20 | -Wredundant-constraints 21 | default-language: Haskell2010 22 | 23 | executable test-package-exe 24 | hs-source-dirs: app 25 | main-is: Main.hs 26 | ghc-options: -Wall 27 | -Wcompat 28 | -Wincomplete-record-updates 29 | -Wincomplete-uni-patterns 30 | -Wredundant-constraints 31 | -threaded -rtsopts -with-rtsopts=-N 32 | build-depends: base 33 | , test-package 34 | default-language: Haskell2010 35 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/src/Language/Haskell/Indexer/Backend/GhcEnv.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE RankNTypes #-} 16 | 17 | module Language.Haskell.Indexer.Backend.GhcEnv 18 | ( GhcEnv(..) 19 | ) where 20 | 21 | import qualified Outputable as GHC 22 | 23 | -- | Contains useful functions for the downstream analyser, without exposing 24 | -- too much GHC internals. 25 | data GhcEnv = GhcEnv 26 | { ghcPrintPrecise :: forall a. (GHC.Outputable a) => a -> String 27 | -- Note: there also seems to be a tidyType family of methods in GHC, but 28 | -- usage is not entirely apparent yet. 29 | , ghcPrintUnqualified :: forall a. (GHC.Outputable a) => a -> String 30 | } 31 | -------------------------------------------------------------------------------- /haskell-indexer-translate/haskell-indexer-translate.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-translate 2 | version: 0.1.0.0 3 | synopsis: Translation layer isolating compiler backends from frontends. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Haskell.Indexer.Translate 18 | Language.Haskell.Indexer.Translate.Render 19 | Language.Haskell.Indexer.Translate.Utils 20 | build-depends: base >=4.8 && <5 21 | , bytestring 22 | , filepath 23 | , text 24 | ghc-options: -Wall 25 | -Wcompat 26 | -Wincomplete-record-updates 27 | -Wincomplete-uni-patterns 28 | -Wredundant-constraints 29 | default-language: Haskell2010 30 | -------------------------------------------------------------------------------- /haskell-indexer-plugin/haskell-indexer-plugin.cabal: -------------------------------------------------------------------------------- 1 | -- Initial haskell-indexer-plugin.cabal generated by cabal init. For 2 | -- further documentation, see http://haskell.org/cabal/users-guide/ 3 | 4 | name: haskell-indexer-plugin 5 | version: 0.1.0.0 6 | -- synopsis: 7 | -- description: 8 | license: Apache-2.0 9 | license-file: LICENSE 10 | author: Matthew Pickering 11 | maintainer: matthewtpickering@gmail.com 12 | -- copyright: 13 | -- category: 14 | build-type: Simple 15 | extra-source-files: CHANGELOG.md 16 | cabal-version: >=1.10 17 | 18 | library 19 | exposed-modules: Haskell.Indexer.Plugin 20 | -- other-modules: 21 | -- other-extensions: 22 | build-depends: base >=4.12 && < 5, ghc 23 | , haskell-indexer-backend-ghc 24 | , haskell-indexer-backend-core 25 | , haskell-indexer-pipeline-ghckythe-wrapper 26 | , haskell-indexer-frontend-kythe 27 | , filepath, directory 28 | hs-source-dirs: src 29 | ghc-options: -Wall 30 | -Wcompat 31 | -Wincomplete-record-updates 32 | -Wincomplete-uni-patterns 33 | -Wredundant-constraints 34 | default-language: Haskell2010 35 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe-wrapper/main/GhcKytheWrapper.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {- | Runs the indexer and emits serialized Kythe storage protos to stdout. 16 | The individual protos are prefixed by a varint-encoded binary length. 17 | 18 | Example: Indexing a module. 19 | 20 | $ ghc_wrapper -c mycorpus -P gcc -- A.hs \ 21 | | /opt/kythe/tools/write_entries -graphstore /tmp/gs 22 | $ /opt/kythe/tools/http_server \ 23 | -graphstore /tmp/gs \ 24 | -listen 0.0.0.0:8000 \ 25 | -public_resources /opt/kythe/web/ui 26 | 27 | Note: the arguments after the '--' are standard GHC arguments. 28 | -} 29 | module Main where 30 | 31 | import Language.Haskell.Indexer.Args (wrapperMain) 32 | 33 | main :: IO () 34 | main = wrapperMain 35 | -------------------------------------------------------------------------------- /kythe-schema/src/Language/Kythe/Schema/Raw/VName.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | module Language.Kythe.Schema.Raw.VName 16 | ( VName(..) 17 | ) where 18 | 19 | import Data.Text (Text) 20 | 21 | -- | A VName (Vector-Name) is the primary unit of naming in the Kythe graph. 22 | -- It is a set of basis facts about a graph node that can uniquely identify it. 23 | data VName = VName 24 | { vnSignature :: !Text -- ^ Opaque signature generated by an analyser. 25 | , vnCorpus :: !Text -- ^ Loosely, a collection of related files. 26 | , vnRoot :: !Text -- ^ Corpus-specific subtree root. Can be empty. 27 | , vnPath :: !Text -- ^ Location relative to the corpus and root. 28 | , vnLanguage :: !Text -- ^ Schema-defined label for supported languages. 29 | } 30 | -------------------------------------------------------------------------------- /serve.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if (($# != 1)); then 3 | echo "Usage: $0 localhost:8080" >&2 4 | echo "Env variables with their defaults:" >&2 5 | echo " - INDEXER_OUTPUT_DIR=/tmp/indexer-output" >&2 6 | echo " Where to read entries from, and write serving tables." >&2 7 | echo " - KYTHE_DIR=/opt/kythe" >&2 8 | echo " Kythe install dir, containing Kythe tools.." >&2 9 | exit 1 10 | fi 11 | 12 | INDEXER_OUTPUT_DIR="${INDEXER_OUTPUT_DIR:-/tmp/indexer-output}" 13 | KYTHE_DIR="${KYTHE_DIR:-/opt/kythe}" 14 | ADDRESS="$1" 15 | 16 | # Serve the index 17 | # =============== 18 | # Delete old Kythe GraphStore and Kythe serving tables. 19 | rm -fr "$INDEXER_OUTPUT_DIR"/{gs,tbl} 20 | 21 | 22 | # It's probably more efficient to cat them together, but this way we see 23 | # if a given one is corrupted for any reason. 24 | for e in "$INDEXER_OUTPUT_DIR"/*.entries 25 | do 26 | echo " * ${e}" 27 | $KYTHE_DIR/tools/write_entries --graphstore "$INDEXER_OUTPUT_DIR/gs" < "$e" 28 | done 29 | echo "== Converting to serving tables." 30 | $KYTHE_DIR/tools/write_tables \ 31 | --graphstore "$INDEXER_OUTPUT_DIR/gs" \ 32 | --out "$INDEXER_OUTPUT_DIR/tbl" \ 33 | --compress_shards 34 | echo "== Starting HTTP server." 35 | echo " * Click the ::/ in the top-left!" 36 | $KYTHE_DIR/tools/http_server \ 37 | --serving_table "$INDEXER_OUTPUT_DIR/tbl" \ 38 | --listen "$ADDRESS" \ 39 | --public_resources $KYTHE_DIR/web/ui 40 | -------------------------------------------------------------------------------- /kythe-schema/kythe-schema.cabal: -------------------------------------------------------------------------------- 1 | name: kythe-schema 2 | version: 0.1.0.0 3 | synopsis: Library for emitting Kythe schema entries. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Kythe.Schema.Raw.VName 18 | Language.Kythe.Schema.Raw.Proto 19 | Language.Kythe.Schema.Raw 20 | Language.Kythe.Schema.Typed 21 | build-depends: base >=4.8 && <5 22 | , bytestring 23 | , data-default 24 | , kythe-proto 25 | , lens-family 26 | , proto-lens >= 0.7 && < 0.8 27 | , text 28 | ghc-options: -Wall 29 | -Wcompat 30 | -Wincomplete-record-updates 31 | -Wincomplete-uni-patterns 32 | -Wredundant-constraints 33 | default-language: Haskell2010 34 | -------------------------------------------------------------------------------- /haskell-indexer-frontend-kythe/haskell-indexer-frontend-kythe.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-frontend-kythe 2 | version: 0.1.0.0 3 | synopsis: Emits Kythe schema based on translation layer data. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Haskell.Indexer.Frontend.Kythe 18 | build-depends: base >=4.8 && <5 19 | , bytestring 20 | , conduit 21 | , haskell-indexer-translate >= 0.1 22 | , kythe-schema >= 0.1 23 | , mmorph 24 | , mtl 25 | , text 26 | , text-offset >= 0.1 27 | , transformers 28 | ghc-options: -Wall 29 | -Wcompat 30 | -Wincomplete-record-updates 31 | -Wincomplete-uni-patterns 32 | -Wredundant-constraints 33 | default-language: Haskell2010 34 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/TypeclassRef.hs: -------------------------------------------------------------------------------- 1 | -- | Haskell's "instance of class" relation is mapped to "extends" in the Kythe 2 | -- schema. The choice is somewhat arbitrary, but enables nice interop with 3 | -- Kythe-based tooling. Mapping visually (and a bit superficially): 4 | -- 5 | -- Haskell class ~ interface 6 | -- Haskell instance ~ (singleton-instantiated) implementation 7 | -- 8 | module TypeClass where 9 | {-# ANN module "HLint: ignore Eta reduce" #-} 10 | 11 | -- - @Foo defines/binding ClassFoo 12 | class Foo a where 13 | -- TODO(robinpalotai): - FunFoo childof ClassFoo 14 | -- - @foo defines/binding FunFoo 15 | foo :: a -> String 16 | 17 | -- For UI convenience, we emit the binding site to the 'instance' keyword, since 18 | -- it is easier to click than say spaces in the instance head. 19 | -- - @instance defines/binding InstIntFoo 20 | -- - @Foo ref ClassFoo 21 | -- - InstIntFoo extends ClassFoo 22 | instance Foo Int where 23 | -- TODO(robinpalotai): - FunIntFoo childof InstIntFoo 24 | -- - @foo defines/binding FunIntFoo 25 | -- - FunIntFoo overrides FunFoo 26 | -- - FunIntFoo overrides/root FunFoo 27 | -- - @show childof FunIntFoo 28 | foo = show 29 | 30 | -- - @foo ref FunFoo 31 | f x = foo x 32 | 33 | g :: Int -> String 34 | -- TODO(robinpalotai): - @foo ref FunIntFoo, instead of the below, when we know 35 | -- the precise instance (here it is coerced by the type signature). 36 | -- - @foo ref FunFoo 37 | g = foo 38 | -------------------------------------------------------------------------------- /stack-example/gen-ghc-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | set -eu 3 | 4 | if [ $# -lt 3 ] 5 | then 6 | echo 7 | echo USAGE 8 | echo 9 | echo " $0 OUTDIR GHC_PATH PLUGIN_DB" 10 | echo 11 | echo " " Generates a wrapper for a ghc compiler in OUTDIR that indexes 12 | echo " " using the given compiler. 13 | echo 14 | echo EXAMPLE 15 | echo 16 | echo " $0 ghc_wrapper" '"$(stack path --compiler-exe)" \' 17 | echo ' "$(stack path --snapshot-pkg-db)"' 18 | echo 19 | exit 1 20 | fi 21 | 22 | RELATIVE_OUTDIR="$1" 23 | mkdir -p "$RELATIVE_OUTDIR" 24 | OUTDIR="$(cd "$RELATIVE_OUTDIR"; pwd)" 25 | GHC="$2" 26 | GHC_DIR="$(dirname "$2")" 27 | PLUGIN_DB="$3" 28 | GHC_VERSION=$("$GHC" --numeric-version) 29 | GHC_WRAPPER="$OUTDIR/ghc-$GHC_VERSION" 30 | 31 | # symlink all executables in the bin folder 32 | for f in $(ls "$GHC_DIR") 33 | do 34 | ln -sf "$GHC_DIR/$f" "$OUTDIR/$f" 35 | done 36 | # Remove the symlink to ghc to avoid modifying the target 37 | rm -f "$GHC_WRAPPER" 38 | # Write the ghc wrapper 39 | cat > "$GHC_WRAPPER" <= 0.7 22 | build-depends: base >=4.8 && <5 23 | , proto-lens-runtime >= 0.7 && < 0.8 24 | ghc-options: -Wall 25 | -Wcompat 26 | -Wincomplete-record-updates 27 | -Wincomplete-uni-patterns 28 | -Wredundant-constraints 29 | default-language: Haskell2010 30 | 31 | custom-setup 32 | setup-depends: base >=4.8 && <5 33 | , Cabal 34 | , proto-lens-protoc >= 0.7 35 | , proto-lens-setup == 0.4.0.4 36 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe/haskell-indexer-pipeline-ghckythe.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-pipeline-ghckythe 2 | version: 0.1.0.0 3 | synopsis: Gets GHC invocation arguments and streams Kythe entries. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | 15 | library 16 | hs-source-dirs: src 17 | exposed-modules: Language.Haskell.Indexer.Pipeline.GhcKythe 18 | build-depends: base >=4.8 && <5 19 | , bytestring 20 | , conduit >= 1.3 21 | , filepath 22 | , haskell-indexer-backend-core >= 0.1 23 | , haskell-indexer-backend-ghc >= 0.1 24 | , haskell-indexer-frontend-kythe >= 0.1 25 | , haskell-indexer-translate >= 0.1 26 | , kythe-schema >= 0.1 27 | , mmorph 28 | , mtl 29 | , text 30 | ghc-options: -Wall 31 | -Wcompat 32 | -Wincomplete-record-updates 33 | -Wincomplete-uni-patterns 34 | -Wredundant-constraints 35 | default-language: Haskell2010 36 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/TypeFam.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE DataKinds #-} 2 | {-# LANGUAGE KindSignatures #-} 3 | {-# LANGUAGE TypeFamilies #-} 4 | 5 | module TypeFam where 6 | 7 | import Data.Kind (Type) 8 | import GHC.TypeNats (Nat) 9 | 10 | -- - @Foo defines/binding FamFoo 11 | -- - @n defines/binding _ 12 | -- - @Nat ref _ 13 | -- - @Type ref _ 14 | type family Foo (n :: Nat) :: Type 15 | 16 | -- It'd be nice to have the following setup, mirroring C++ template 17 | -- specialization: 18 | -- 19 | -- * @Foo defines/binding FooFam 20 | -- * FooFam.type abs 21 | -- type family Foo a (n :: Nat) :: Type 22 | -- 23 | -- * @Foo defines/binding FooInt4 24 | -- * FooInt4 specializes AppFooInt4 25 | -- * AppFooInt4.node/kind tapp 26 | -- * AppFooInt4 param.0 FooFam 27 | -- * AppFooInt4 param.1 Four 28 | -- * Four.node/kind constant 29 | -- * Four.text 4 30 | -- type instance Foo Int 4 = Bool 31 | -- 32 | -- * @Foo ref FooInt4 33 | -- type F'oo4 = Foo Int 4 34 | -- 35 | -- * @Foo defines/binding Foo_5 36 | -- type instance Foo a 5 = a 37 | -- 38 | -- * @Foo ref FooString5 39 | -- * @String ref StringTy 40 | -- type F'oo5 = Foo String 5 41 | -- 42 | -- * FooString5 instantiates AppFoo_5String 43 | -- * AppFoo_5String param.0 Foo_5 44 | -- * AppFoo_5String param.1 StringTy 45 | -- 46 | -- * FooString5 specializes AppFooString5 47 | -- * AppFooString5 param.0 FooFam 48 | -- * AppFooString5 param.1 StringTy 49 | -- * AppFooString5 param.2 _ 50 | 51 | -- Instead we have that instance declarations reference the family, for now. 52 | 53 | -- - @Foo ref FamFoo 54 | type instance Foo 4 = Int 55 | 56 | -- - @Foo ref FamFoo 57 | type F'oo4 = Foo 4 58 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/DataRef.hs: -------------------------------------------------------------------------------- 1 | module DataRef where 2 | 3 | -- TODO(robinpalotai): - CtorA childof TypeD 4 | -- TODO(robinpalotai): - CtorB childof TypeD 5 | -- TODO(robinpalotai): maybe have `FieldInt childof CtorB` with an implicit 6 | -- anchor (or an overloaded anchor @Int) defines/binding it. 7 | -- - @D defines/binding TypeD 8 | -- - @A defines/binding CtorA 9 | -- - @B defines/binding CtorB 10 | -- - @Int ref _ 11 | data D = A | B Int 12 | 13 | -- - @D ref TypeD 14 | f :: D -> Int 15 | -- - @A ref CtorA 16 | f A = 0 17 | -- No reference from 'x' to the field, since Kythe only adds references for 18 | -- explicitly referred nodes. 19 | -- - @B ref CtorB 20 | -- - @x defines/binding PatternX 21 | f (B x) = 22 | -- - @x ref PatternX 23 | x 24 | 25 | -- - @Record defines/binding TypeR 26 | data Record = 27 | -- TODO(robinpalotai): - CtorR childof TypeR 28 | -- TODO(robinpalotai): - FieldR childof CtorR 29 | -- TODO(robinpalotai): - FieldR childof TypeR, at least for fields that are 30 | -- shared by all constructors? Tools would like it probably. 31 | -- - @Record defines/binding CtorR 32 | -- - @recInt defines/binding FieldR 33 | Record { recInt :: Int } 34 | 35 | -- - @Record ref TypeR 36 | g :: Record -> Int 37 | -- - @Record ref CtorR 38 | g (Record x) = x 39 | 40 | -- - @recInt ref FieldR 41 | h = recInt 42 | 43 | -- Banging fields (and some other things) makes the Wrapper Id to be used 44 | -- instead of the Worker / DataCon Id. If we don't handle this, we wouldn't 45 | -- be able to find reference to the data constructor. 46 | -- - @MX defines/binding CtorMX 47 | data X a = MX { unX :: !a } 48 | 49 | -- - @MX ref CtorMX 50 | makeX = MX 5 51 | -------------------------------------------------------------------------------- /kythe-verification/README.md: -------------------------------------------------------------------------------- 1 | # Kythe verification 2 | 3 | The Kythe verifier parses expectation-annotated source code, and correlates it 4 | with the Kythe entries generated from that source. 5 | See [Kythe verifier docs](http://kythe.io/docs/kythe-verifier.html) and 6 | [testing a new indexer](http://kythe.io/docs/schema/writing-an-indexer.html#_testing). 7 | 8 | Aim to keep the verifier sources small, as the unification gets slow pretty fast. 9 | Verify only the aspects you care about. 10 | 11 | ## Running the verifier. 12 | 13 | The verifier script expects `ghc_kythe_wrapper` to be on the `PATH`. 14 | 15 | For example, if you use stack, do `stack install` in the base directory. 16 | 17 | The script also expects the Kythe verifier tool to be in 18 | `/opt/kythe/tools/verifier`. Please look into the script to change, or fix 19 | [#24](https://github.com/google/haskell-indexer/issues/24). 20 | 21 | Change to the `kythe-verification` directory and run `./test.sh`. 22 | 23 | ## Fixing errors. 24 | 25 | In case of error, you'll get something like: 26 | 27 | ``` 28 | Verifying: testdata/basic/TypeDef.hs 29 | Could not verify all goals. The furthest we reached was: 30 | testdata/basic/TypeDef.hs:4:6-4:7 @ 31 | ``` 32 | 33 | This means that the doc assert at line 4, around cols 6-7 was not satisfied. 34 | 35 | ### Some hints 36 | 37 | #### Did you put a span anchor assertion on the right line? 38 | 39 | For example, 40 | 41 | ``` 42 | -- - @foo defines/binding Something 43 | -- - @bar ref ARef 44 | foo = bar 45 | ``` 46 | 47 | works, but if you interleave a non-assertion comment, then it already won't 48 | find the anchor: 49 | 50 | ``` 51 | -- - @foo ... 52 | -- - @bar ... 53 | -- I'm a comment. 54 | foo = bar 55 | -------------------------------------------------------------------------------- /nix/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | }: 3 | 4 | with { 5 | haskellPackages = pkgs.haskellPackages.override (old: { 6 | overrides = 7 | with pkgs; 8 | with { 9 | indexerSrc = fetchgit { 10 | url = "https://github.com/google/haskell-indexer"; 11 | sha256 = "0rr32birxd5xw85jyjx2q3dfgv7szpd142hbwf6isgiv3a3bhkx8"; 12 | rev = "08aed342bc76315534e7ee06ed497689276279e6"; 13 | }; 14 | }; 15 | lib.composeExtensions (old.overrides or (_: _: {})) (self: super: rec { 16 | kythe-proto = haskell.lib.addBuildDepend (super.callPackage ./kythe-proto.nix { inherit indexerSrc; }) [protobuf]; 17 | kythe-schema = super.callPackage ./kythe-schema.nix { inherit indexerSrc; }; 18 | text-offset = super.callPackage ./text-offset.nix { inherit indexerSrc; }; 19 | haskell-indexer-backend-core = super.callPackage ./haskell-indexer-backend-core.nix { inherit indexerSrc; }; 20 | haskell-indexer-backend-ghc = super.callPackage ./haskell-indexer-backend-ghc.nix { inherit indexerSrc; }; 21 | haskell-indexer-frontend-kythe = super.callPackage ./haskell-indexer-frontend-kythe.nix { inherit indexerSrc; }; 22 | haskell-indexer-pathutil = super.callPackage ./haskell-indexer-pathutil.nix { inherit indexerSrc; }; 23 | haskell-indexer-pipeline-ghckythe = super.callPackage ./haskell-indexer-pipeline-ghckythe.nix { inherit indexerSrc; }; 24 | haskell-indexer-pipeline-ghckythe-wrapper = super.callPackage ./haskell-indexer-pipeline-ghckythe-wrapper.nix { inherit indexerSrc; }; 25 | haskell-indexer-translate = super.callPackage ./haskell-indexer-translate.nix { inherit indexerSrc; }; 26 | }); 27 | }); 28 | }; 29 | { 30 | indexer = haskellPackages.haskell-indexer-pipeline-ghckythe-wrapper; 31 | } 32 | -------------------------------------------------------------------------------- /kythe-verification/testdata/basic/RecordReadRef.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE NamedFieldPuns, RecordWildCards #-} 2 | module RecordReadRef where 3 | {-# ANN module "HLint: ignore Eta reduce" #-} 4 | 5 | -- TODO(robinpalotai): define and refer separate bindings for accessors. Maybe 6 | -- add an option to choose if the bindings will overlap, or if one should go 7 | -- on the ::s. So can be switched depending on overlap support from tools. 8 | -- Or this can just be a hint to the frontend (assuming backend provides the 9 | -- alternative span). 10 | -- - @foo defines/binding FieldFoo 11 | -- - @bar defines/binding FieldBar 12 | -- - @#1Rec defines/binding CtorR 13 | data Rec = Rec { foo :: Int, bar :: Int } 14 | 15 | -- No field references here, only to the introduced local binding (see 16 | -- DataRef.hs for more comments). 17 | unpack (Rec f b) = f 18 | 19 | -- - @foo ref FieldFoo 20 | access r = foo r 21 | 22 | -- TODO(robinpalotai): revamp record references, and omit field refs originating 23 | -- from wildcard locations (since they are not explicit references?) Also 24 | -- elaborate this section here (AST vs interpretation). What does this being 25 | -- a syntactic sugar imply? 26 | -- TODO(robinpalotai): maybe fix the spans of the wildcard matches to only the 27 | -- double-dots. 28 | -- - @Rec ref CtorR 29 | -- - @"Rec{..}" ref FieldFoo 30 | -- - @"Rec{..}" ref FieldBar 31 | -- - @foo ref FieldFoo 32 | wildcard Rec{..} = foo 33 | 34 | -- - @foo ref FieldFoo 35 | punned Rec{foo} = 36 | -- - @foo ref FieldFoo 37 | foo 38 | 39 | -- - @foo ref FieldFoo 40 | -- - @baz defines/binding VarBaz 41 | reassigned Rec{foo=baz} = 42 | -- - @baz ref VarBaz 43 | baz 44 | 45 | -- - @bar ref FieldBar 46 | -- - @"Rec{bar,..}" ref FieldFoo 47 | rightWild Rec{bar,..} = 48 | -- - @foo ref FieldFoo 49 | -- - @bar ref FieldBar 50 | foo + bar 51 | -------------------------------------------------------------------------------- /wrappers/stack/ghc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # GHC wrapper for indexing Haskell packages. 6 | # Note that variables INDEXER_OUTPUT_DIR and REALGHC are set outside this script. 7 | 8 | log() { 9 | echo "$1" >> "$INDEXER_OUTPUT_DIR/$PKG.log" 10 | } 11 | 12 | log "========= FAKE GHC =======" 13 | log " == pwd: $PWD" 14 | log "== Passing through.." 15 | log "$REALGHC $*" 16 | $REALGHC "$@" 17 | RESULT=$? 18 | # $(stack path --compiler_exe) is invoked by `stack build` multiple times, 19 | # we are only interested when `--make` is specified. 20 | if [[ "${@#--make}" != "$@" ]]; then 21 | PKG=${PWD##*/} 22 | log " == pkg: $PKG" 23 | EXE_FOUND=false 24 | # GHC is invoked twice for executable modules, first with "-no-link" but with 25 | # no "-o" argument specified, this flag prevents that run from overwriting 26 | # librarie entries file 27 | NO_LINK=false 28 | for i in "$@" 29 | do 30 | if [ "$EXE_FOUND" = true ] 31 | then 32 | EXE="$(basename $i)" 33 | EXE_SUFFIX="-$EXE" 34 | RENAME_MAIN="--rename_main $PKG$EXE_SUFFIX" 35 | log " == exe-suffix: $EXE_SUFFIX" 36 | break 37 | elif [ "$i" = "-o" ]; then 38 | EXE_FOUND=true 39 | elif [ "$i" = "-no-link" ]; then 40 | NO_LINK=true 41 | fi 42 | done 43 | log "== Invoking indexer" 44 | PATH_NO_EXT="$INDEXER_OUTPUT_DIR/${PKG}${EXE_SUFFIX}" 45 | if [ "$NO_LINK" = true ] 46 | then 47 | ENTRIES_FILE=/dev/null 48 | else 49 | ENTRIES_FILE="$PATH_NO_EXT.entries" 50 | fi 51 | log "== Output entries file: $ENTRIES_FILE" 52 | if ! ghc_kythe_wrapper \ 53 | --corpus haskell \ 54 | --drop_path_prefix './' \ 55 | --prepend_path_prefix "$PKG/" \ 56 | $RENAME_MAIN \ 57 | -- \ 58 | "$@" > "$ENTRIES_FILE" 2>> "$PATH_NO_EXT.stderr"; then 59 | echo "${PKG}${EXE_SUFFIX} had error" >> "$INDEXER_OUTPUT_DIR/errors" 60 | fi 61 | fi 62 | exit $RESULT 63 | -------------------------------------------------------------------------------- /haskell-indexer-pathutil/src/Language/Haskell/Indexer/Util/Path.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | module Language.Haskell.Indexer.Util.Path 16 | ( asTextPath 17 | , stripTmpPrefix 18 | ) where 19 | 20 | import Data.List (isPrefixOf) 21 | import qualified Data.Text as T 22 | import System.FilePath (splitPath, joinPath) 23 | 24 | -- | Removes the workdir prefix from likely temporary paths. 25 | -- Useful for 'aoFilePathTransform' in 'AnalysisOptions'. 26 | stripTmpPrefix :: FilePath -> FilePath 27 | stripTmpPrefix path = case partsAfterTmp path of 28 | Just (rand:ps) | isValidRandomDir rand -> joinPath ps 29 | _ -> path 30 | where 31 | -- | Finds the longest suffix of the path that begins with "tmp/". 32 | partsAfterTmp :: FilePath -> Maybe [String] 33 | partsAfterTmp = go . splitPath 34 | where go [] = Nothing 35 | go ("tmp/":rest) = Just rest 36 | go (_:rest) = go rest 37 | -- | True if the directory matches some known random-generation pattern. 38 | isValidRandomDir :: String -> Bool 39 | isValidRandomDir dir = all (`elem` '/':['0'..'9']) dir 40 | || "tempfile" `isPrefixOf` dir 41 | 42 | -- | Wraps a FilePath operation as a Text operation. 43 | asTextPath :: (FilePath -> FilePath) -> T.Text -> T.Text 44 | asTextPath f = T.pack . f . T.unpack 45 | -------------------------------------------------------------------------------- /kythe-schema/src/Language/Kythe/Schema/Raw/Proto.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE OverloadedStrings #-} 16 | -- | Serializes the raw schema types to proto equivalent. 17 | module Language.Kythe.Schema.Raw.Proto 18 | ( toEntryProto 19 | ) where 20 | 21 | import Data.ProtoLens 22 | import Lens.Family2 ((&), (.~)) 23 | 24 | import qualified Language.Kythe.Schema.Raw as Raw 25 | -- TODO(robinpalotai): adapt path for open-source release. 26 | import qualified Proto.Kythe.Proto.Storage as K 27 | import qualified Proto.Kythe.Proto.Storage_Fields as K 28 | 29 | toEntryProto :: Raw.Entry -> K.Entry 30 | toEntryProto (Raw.EdgeE (Raw.Edge srcVName edgeName targetVName)) = defMessage 31 | & K.source .~ toVNameProto srcVName 32 | & K.edgeKind .~ edgeName 33 | & K.target .~ toVNameProto targetVName 34 | & K.factName .~ "/" -- expected by Kythe tools. 35 | toEntryProto (Raw.FactE (Raw.Fact vname factName factValue)) = defMessage 36 | & K.source .~ toVNameProto vname 37 | & K.factName .~ factName 38 | & K.factValue .~ factValue 39 | 40 | toVNameProto :: Raw.VName -> K.VName 41 | toVNameProto (Raw.VName sig corpus root path lang) = defMessage 42 | & K.signature .~ sig 43 | & K.corpus .~ corpus 44 | & K.root .~ root 45 | & K.path .~ path 46 | & K.language .~ lang 47 | -------------------------------------------------------------------------------- /text-offset/text-offset.cabal: -------------------------------------------------------------------------------- 1 | name: text-offset 2 | version: 0.1.0.0 3 | synopsis: Library for converting between line/column and byte offset. 4 | description: Translates visual line/column info (as from editor or 5 | warning message) to byte offset. Assumes UTF-8 encoding. 6 | homepage: https://github.com/google/haskell-indexer 7 | license: Apache-2.0 8 | license-file: LICENSE 9 | author: Robin Palotai 10 | maintainer: palotai.robin@gmail.com 11 | copyright: Google Inc. 12 | category: Text 13 | build-type: Simple 14 | cabal-version: 2.0 15 | 16 | library 17 | hs-source-dirs: src 18 | exposed-modules: Data.Text.Offset 19 | build-depends: base >= 4.7 && < 4.15 20 | , text >= 1.2.0.6 && < 1.3 21 | , vector >= 0.10.2.3 && < 0.13 22 | ghc-options: -Wall 23 | -Wcompat 24 | -Wincomplete-record-updates 25 | -Wincomplete-uni-patterns 26 | -Wredundant-constraints 27 | default-language: Haskell2010 28 | 29 | Test-Suite offset_test 30 | default-language: Haskell2010 31 | type: exitcode-stdio-1.0 32 | main-is: OffsetTest.hs 33 | hs-source-dirs: tests 34 | build-depends: base 35 | , bytestring >= 0.10.4 && < 0.11 36 | , test-framework >= 0.8.1.1 && < 0.9 37 | , test-framework-hunit >= 0.3.0.1 && < 0.4 38 | , test-framework-quickcheck2 >= 0.3.0.3 && < 0.4 39 | , text 40 | , text-offset 41 | , HUnit >= 1.2.5.2 && < 1.7 42 | , QuickCheck >= 2.7.6 && < 2.14 43 | ghc-options: -Wall 44 | -Wcompat 45 | -Wincomplete-record-updates 46 | -Wincomplete-uni-patterns 47 | -Wredundant-constraints 48 | -------------------------------------------------------------------------------- /kythe-schema/src/Language/Kythe/Schema/Raw.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-| Raw Kythe schema, suitable for emitting from an analyser. 16 | 17 | See http://www.kythe.io/docs/kythe-storage.html and 18 | https://kythe.io/docs/schema/writing-an-indexer.html#_modeling_kythe_entries. 19 | The values in this structure are semantically restricted by the Kythe schema 20 | http://www.kythe.io/docs/schema/. Note: see the head of the github repo for the 21 | latest schema docs, as it is still in flux a bit. 22 | 23 | This module is suggested to be imported qualified, and most client code rather 24 | use the safer 'Language.Kythe.Schema.Typed' module. 25 | -} 26 | module Language.Kythe.Schema.Raw 27 | ( Entry(..) 28 | , Edge(..), Fact(..) 29 | , EdgeName, FactName, FactValue 30 | , module Language.Kythe.Schema.Raw.VName 31 | ) where 32 | 33 | import Data.Text (Text) 34 | import Data.ByteString (ByteString) 35 | 36 | import Language.Kythe.Schema.Raw.VName (VName(..)) 37 | 38 | -- | Top-level Kythe analyser artifact. 39 | data Entry 40 | = EdgeE !Edge 41 | | FactE !Fact 42 | 43 | -- | Glorified pair of strings, in the context of a given node (identified by 44 | -- its VName). 45 | data Fact = Fact !VName !FactName !FactValue 46 | 47 | -- | Directed edge between nodes. 48 | data Edge = Edge !VName !EdgeName !VName -- TODO(robinpalotai): ordinal 49 | 50 | type EdgeName = Text 51 | type FactName = Text 52 | type FactValue = ByteString 53 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/src/Language/Haskell/Indexer/Backend/GhcArgs.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | {-# Language StrictData #-} 15 | module Language.Haskell.Indexer.Backend.GhcArgs 16 | ( GhcArgs(..) 17 | , ToolOverride(..) 18 | , LibdirOverride(..) 19 | , defaultGhcArgs 20 | ) where 21 | 22 | -- | Args to call GHC with, to perform the compilation. 23 | data GhcArgs = GhcArgs 24 | { gaToolOverride :: ToolOverride 25 | , gaArgs :: [String] 26 | , gaLibdirOverride :: Maybe LibdirOverride 27 | } 28 | 29 | -- | Lets selective override of various tools used during compilation. 30 | data ToolOverride = ToolOverride 31 | { overridePgmP :: Maybe FilePath 32 | -- ^ Override the preprocessor if needed. Only overrides the binary, but 33 | -- keeps whatever flags were set on it originally (from the GHC command 34 | -- line). 35 | } 36 | 37 | -- | The libdir determines the default libraries available to GHC. Normally it 38 | -- is determined by calling a ghc-paths library function, but under special 39 | -- circumstances one might want to override it. 40 | data LibdirOverride 41 | = AddPrefixToLibdir FilePath 42 | -- ^ Prepend prefix to libdir. 43 | | OverrideLibdir FilePath 44 | -- ^ Explicit override. 45 | 46 | defaultGhcArgs :: GhcArgs 47 | defaultGhcArgs = GhcArgs 48 | { gaToolOverride = ToolOverride 49 | { overridePgmP = Nothing 50 | } 51 | , gaArgs = [] 52 | , gaLibdirOverride = Nothing 53 | } 54 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe-wrapper/haskell-indexer-pipeline-ghckythe-wrapper.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-pipeline-ghckythe-wrapper 2 | version: 0.1.0.0 3 | synopsis: Executable able to take GHC arguments and emitting Kythe entries. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | Should be merged into ghckythe eventually. 6 | homepage: https://github.com/google/haskell-indexer 7 | license: Apache-2.0 8 | license-file: LICENSE 9 | author: Robin Palotai 10 | maintainer: robinpalotai@google.com 11 | copyright: Google Inc. 12 | category: Language 13 | build-type: Simple 14 | cabal-version: 2.0 15 | 16 | library 17 | hs-source-dirs: src 18 | build-depends: base >=4.8 && <5 19 | , bytestring 20 | , ghc 21 | , haskell-indexer-backend-core >= 0.1 22 | , haskell-indexer-backend-ghc >= 0.1 23 | , haskell-indexer-pathutil >= 0.1 24 | , haskell-indexer-pipeline-ghckythe >= 0.1 25 | , haskell-indexer-translate >= 0.1 26 | , kythe-schema >= 0.1 27 | , optparse-applicative 28 | , proto-lens >= 0.7 && < 0.8 29 | , text 30 | exposed-modules: Language.Haskell.Indexer.Args 31 | ghc-options: -Wall 32 | -Wcompat 33 | -Wincomplete-record-updates 34 | -Wincomplete-uni-patterns 35 | -Wredundant-constraints 36 | default-language: Haskell2010 37 | 38 | 39 | executable "ghc_kythe_wrapper" 40 | hs-source-dirs: main 41 | main-is: GhcKytheWrapper.hs 42 | build-depends: base >= 4.8 && < 5, haskell-indexer-pipeline-ghckythe-wrapper 43 | ghc-options: -Wall 44 | -Wcompat 45 | -Wincomplete-record-updates 46 | -Wincomplete-uni-patterns 47 | -Wredundant-constraints 48 | default-language: Haskell2010 49 | -------------------------------------------------------------------------------- /haskell-indexer-translate/src/Language/Haskell/Indexer/Translate/Utils.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE OverloadedStrings #-} 16 | {-# LANGUAGE RecordWildCards #-} 17 | module Language.Haskell.Indexer.Translate.Utils 18 | ( spanTickFragment -- TODO(robinpalotai): rename? 19 | , tickString 20 | ) where 21 | 22 | import Control.Monad (guard) 23 | import qualified Data.Text as T 24 | import qualified Data.Text.Lazy as TL 25 | import qualified Data.Text.Lazy.Builder as TB 26 | import qualified Data.Text.Lazy.Builder.Int as TB 27 | 28 | import Language.Haskell.Indexer.Translate 29 | ( SourcePath(..), Span(..), Pos(..), Tick(..), PkgModule(..) ) 30 | 31 | -- | Returns a fragment usable in a TickString for a given Span. 32 | -- Includes char-based line/col positions and the file name. 33 | -- 34 | -- Example output: some/path/to.hs:(3,1)-(3,4) 35 | spanTickFragment :: Span -> T.Text 36 | spanTickFragment (Span (Pos l1 c1 path) (Pos l2 c2 _)) = 37 | TL.toStrict . TB.toLazyText . mconcat $ 38 | [ TB.fromText (unSourcePath path) 39 | , ":(", TB.decimal l1, ",", TB.decimal c1 40 | , ")-(", TB.decimal l2, ",", TB.decimal c2, ")" 41 | ] 42 | 43 | -- | Assuming that package+module name are unique, returns a unique string for 44 | -- a Tick. 45 | tickString :: Tick -> T.Text 46 | tickString Tick{..} = T.intercalate ":" $ 47 | [ "haskell" 48 | , if tickTermLevel then "term" else "type" 49 | , getPackage tickPkgModule 50 | , getModule tickPkgModule 51 | , tickThing 52 | ] 53 | ++ (guard (not tickUniqueInModule) >> 54 | [ maybe "?" spanTickFragment tickSpan ]) 55 | -------------------------------------------------------------------------------- /kythe-verification/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # TODO(robinp): take these from outer env if present. 18 | GHC_KYTHE="${GHC_KYTHE:-ghc_kythe_wrapper}" 19 | KYTHE_DIR="${KYTHE_DIR:-/opt/kythe}" 20 | VERIFIER="${VERIFIER:-$KYTHE_DIR/tools/verifier}" 21 | 22 | BASIC="testdata/basic" 23 | 24 | RESULT=0 25 | 26 | die() { 27 | RESULT=-1 28 | echo "*********************************" 29 | echo "* THERE WAS AN ERROR, SEE ABOVE *" 30 | echo "*********************************" 31 | } 32 | 33 | if [[ ! -f "$(which $GHC_KYTHE)" ]]; then 34 | echo "* GHC-Kythe wrapper [$GHC_KYTHE] not found! See Readme." 35 | exit -1 36 | fi 37 | 38 | if [[ ! -f "$VERIFIER" ]]; then 39 | echo "* Kythe verifier [$VERIFIER] not found! See Readme." 40 | exit -1 41 | fi 42 | 43 | for fut in \ 44 | "$BASIC/Anchors.hs" \ 45 | "$BASIC/CrossRef1.hs $BASIC/CrossRef2.hs" \ 46 | "$BASIC/DataRef.hs" \ 47 | "$BASIC/DocUri.hs" \ 48 | "$BASIC/FunctionArgRef.hs" \ 49 | "$BASIC/ImpExpDefs.hs $BASIC/ImportRefs.hs" \ 50 | "$BASIC/LocalRef.hs" \ 51 | "$BASIC/Module.hs" \ 52 | "$BASIC/ModuleDocUri.hs" \ 53 | "$BASIC/PatSyn.hs" \ 54 | "$BASIC/RecordReadRef.hs" \ 55 | "$BASIC/RecordWriteRef.hs" \ 56 | "$BASIC/RecursiveRef.hs" \ 57 | "$BASIC/TypeclassRef.hs" \ 58 | "$BASIC/TypeDef.hs" \ 59 | "$BASIC/TypeFam.hs" \ 60 | "$BASIC/DataFam.hs" \ 61 | "$BASIC/ClosedFam.hs" \ 62 | "$BASIC/AssocType.hs" \ 63 | "$BASIC/TypeOperators.hs" \ 64 | "$BASIC/TypeVarInSig.hs" 65 | do 66 | echo "Verifying: $fut" 67 | $GHC_KYTHE -- $fut 2> /dev/null | $VERIFIER -goal_prefix '-- -' --check_for_singletons=true --ignore_dups $fut \ 68 | || die 69 | done 70 | 71 | exit $RESULT 72 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Build 2 | permissions: read-all 3 | 4 | on: 5 | push: 6 | branches: 7 | - 'master' 8 | 9 | pull_request: 10 | paths: 11 | - '**.hs' 12 | - '**.cabal' 13 | - 'stack*.yaml' 14 | - '.github/workflows/build.yaml' 15 | 16 | jobs: 17 | cabal: 18 | strategy: 19 | matrix: 20 | os: 21 | - "ubuntu-latest" 22 | 23 | ghc: 24 | - "8.10.7" 25 | 26 | name: cabal / ${{ matrix.os }} / ghc ${{ matrix.ghc }} 27 | runs-on: ${{ matrix.os }} 28 | 29 | steps: 30 | - name: Checkout 31 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 32 | 33 | - name: Cache dependencies 34 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 35 | with: 36 | path: | 37 | ~/.cabal 38 | ~/.ghcup 39 | key: cabal-${{ matrix.os }}-${{ matrix.ghc }}-${{ hashFiles('*/*.cabal') }} 40 | 41 | - name: Set up protoc 42 | uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 43 | 44 | - name: Set up Haskell 45 | uses: haskell-actions/setup@bbd90a29996ac33b1c644a42206e312fc0379748 # v2.7.9 46 | with: 47 | ghc-version: ${{ matrix.ghc }} 48 | 49 | - name: Configure Cabal 50 | run: cabal configure --enable-tests --enable-benchmarks 51 | 52 | - name: Build 53 | run: cabal build all 54 | 55 | - name: Test 56 | run: cabal test all 57 | 58 | stack: 59 | runs-on: "ubuntu-latest" 60 | 61 | steps: 62 | - name: Checkout 63 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 64 | 65 | - name: Cache dependencies 66 | uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 67 | with: 68 | path: | 69 | ~/.ghcup 70 | ~/.stack 71 | key: stack-${{ hashFiles('stack.yaml', '*/*.cabal') }} 72 | 73 | - name: Set up protoc 74 | uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 75 | 76 | - name: Set up Haskell 77 | uses: haskell-actions/setup@bbd90a29996ac33b1c644a42206e312fc0379748 # v2.7.9 78 | with: 79 | enable-stack: true 80 | 81 | - name: Build 82 | run: stack build --test --bench --no-run-tests --no-run-benchmarks 83 | 84 | - name: Test 85 | run: stack build --test --bench --no-run-benchmarks 86 | -------------------------------------------------------------------------------- /haskell-indexer-backend-core/src/Language/Haskell/Indexer/Backend/AnalysisOptions.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE OverloadedStrings #-} 16 | 17 | {- | Mostly compiler-agnostic options that can affect the analysis. -} 18 | module Language.Haskell.Indexer.Backend.AnalysisOptions 19 | ( AnalysisOptions(..) 20 | , defaultAnalysisOptions 21 | ) where 22 | 23 | import Data.Text (Text) 24 | 25 | -- | External factors affecting the analysis. 26 | data AnalysisOptions = AnalysisOptions 27 | { aoMainPkgFallback :: !Text 28 | -- ^ GHC needs package = 'main' internally when compiling binaries with a 29 | -- Main module, but we would like to report distinct packages to the 30 | -- frontend. So if the package reported by GHC is 'main', this gets 31 | -- substituted. 32 | -- Note: this is GHC specific, but is not strictly needed by GHC, rather 33 | -- by sane output. Thus here. 34 | -- TODO(robinpalotai): What happens with multiple executables per package? 35 | -- Should we invent a virtual module name based on 36 | -- path for example? 37 | , aoDemanglePackageName :: !(Text -> Text) 38 | -- ^ If the packages name extracted from the build system are mangled for 39 | -- some reason, this function can be used to demangle them. 40 | , aoFilePathTransform :: !(Text -> Text) 41 | -- ^ Can be used to transform filepaths if needed. For example, can be 42 | -- used to drop prefixes for temporary workdirs, or corpus-wide static 43 | -- prefixes. The result should be closely aligned with what programmers 44 | -- recognize as the ideal path for any given file. The transformed path 45 | -- will be the one referenced by frontends when emitting data. 46 | } 47 | 48 | defaultAnalysisOptions :: AnalysisOptions 49 | defaultAnalysisOptions = AnalysisOptions "main" id id 50 | -------------------------------------------------------------------------------- /haskell-indexer-plugin/src/Haskell/Indexer/Plugin.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE BangPatterns #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | module Haskell.Indexer.Plugin where 4 | 5 | import GhcPlugins hiding (($$)) 6 | import TcRnTypes 7 | 8 | import TcRnMonad 9 | import TcRnDriver 10 | import GhcMonad 11 | import Data.IORef 12 | 13 | import Language.Haskell.Indexer.Backend.Ghc 14 | import Language.Haskell.Indexer.Backend.GhcEnv 15 | import Language.Haskell.Indexer.Args 16 | 17 | import System.IO 18 | import System.FilePath 19 | import System.Directory 20 | 21 | -- Defining a 22 | -- which can be dynamically loaded when running GHC with the `-plugin` flag. 23 | plugin :: Plugin 24 | plugin = defaultPlugin 25 | { renamedResultAction = keepRenamedSource 26 | , typeCheckResultAction = install } 27 | 28 | -- Makes a unique path for each module so that we don't clobber each the 29 | -- entries when compiling different modules. The name of the file isn't 30 | -- important as long as it is different for each one. 31 | mkPath :: FilePath -> Module -> FilePath 32 | mkPath fp m 33 | = fp (moduleNameString (moduleName m) ++ (show (moduleUnitId m))) 34 | <.> "entries" 35 | 36 | -- This function is invoked once at the end of compiling every module. 37 | -- It runs the indexer and then outputs the entries to a directory. 38 | install :: [CommandLineOption] -> ModSummary -> TcGblEnv -> TcM TcGblEnv 39 | install opts ms tc_gbl = do 40 | dflags <- getDynFlags 41 | hsc_env <- env_top <$> getEnv 42 | session <- Session <$> liftIO (newIORef hsc_env) 43 | flags <- liftIO (wrapperParser opts) 44 | let outdir = case flagOutput flags of 45 | Nothing -> "haskell-indexer" 46 | Just o -> o 47 | outpath = mkPath outdir (ms_mod ms) 48 | liftIO $ createDirectoryIfMissing False outdir 49 | h <- liftIO (openFile outpath WriteMode) 50 | let env = GhcEnv (showSDoc dflags . ppr) 51 | (showSDocForUser dflags neverQualify . ppr) 52 | action aoes = 53 | unGhc (analyseTypechecked' env aoes ms 54 | (getRenamedStuff tc_gbl) 55 | (tcg_binds tc_gbl) 56 | -- TODO: Fill in the Nothing with the correct span 57 | (ms_mod ms, Nothing)) 58 | session 59 | liftIO $ kythePlugin h action flags 60 | return tc_gbl 61 | 62 | -------------------------------------------------------------------------------- /haskell-indexer-translate/src/Language/Haskell/Indexer/Translate/Render.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {- Utilities for rendering entities from Translate into strings. 16 | Frontends can use this module as a convenience, but it is not mandatory to 17 | stick to the formats if they don't want. 18 | -} 19 | {-# LANGUAGE CPP #-} 20 | {-# LANGUAGE OverloadedStrings #-} 21 | {-# LANGUAGE RecordWildCards #-} 22 | module Language.Haskell.Indexer.Translate.Render 23 | ( tickQualifiedThing 24 | , tickPredictableQualifiedName 25 | , makeDeclIdentifier 26 | , declPreferredUiSpan 27 | ) where 28 | 29 | import Control.Applicative ((<|>)) 30 | import Control.Monad (guard) 31 | #if !MIN_VERSION_base(4,11,0) 32 | import Data.Monoid ((<>)) 33 | #endif 34 | import Data.Text (Text) 35 | 36 | import Language.Haskell.Indexer.Translate 37 | 38 | -- | For informal use only (displaying on UI). 39 | tickQualifiedThing :: Tick -> Text 40 | tickQualifiedThing Tick{..} = 41 | let local = if not tickUniqueInModule then "." else "" 42 | in pmText tickPkgModule <> local <> "." <> tickThing 43 | 44 | -- | A predictable (expectedly stable across reindexings) qualified name. 45 | -- Only present for top-level things. 46 | tickPredictableQualifiedName :: Tick -> Maybe Text 47 | tickPredictableQualifiedName Tick{..} = do 48 | guard tickUniqueInModule 49 | return $! pmText tickPkgModule <> "." <> tickThing 50 | 51 | -- | Creates an informal unqualified name for the Decl suitable for UI display. 52 | -- Prefixes instance methods with the instance name for readability. 53 | makeDeclIdentifier :: Decl -> Text 54 | makeDeclIdentifier d = 55 | let ident = tickThing (declTick d) 56 | in case declExtra d >>= methodForInstance of 57 | Just instName -> instName <> "." <> ident 58 | Nothing -> ident 59 | 60 | -- | UIs may use this span as a main anchor for the decl - for hyperlinking 61 | -- referents and other properties. 62 | declPreferredUiSpan :: Decl -> Maybe Span 63 | declPreferredUiSpan decl = 64 | (declExtra decl >>= alternateIdSpan) <|> declIdentifierSpan decl 65 | 66 | pmText :: PkgModule -> Text 67 | pmText pm = getPackage pm <> ":" <> getModule pm 68 | -------------------------------------------------------------------------------- /stack-example/README.md: -------------------------------------------------------------------------------- 1 | This setup shows how to index haskell packages using `stack` 2 | and the [haskell-indexer-plugin][haskell-indexer-plugin]. 3 | 4 | [haskell-indexer-plugin]: https://github.com/google/haskell-indexer 5 | 6 | First change directory to `stack-example` and build the plugin. 7 | ```bash 8 | $ cd stack-example 9 | $ stack build haskell-indexer-plugin 10 | ``` 11 | 12 | Then generate a ghc wrapper script. This script will pass 13 | to GHC all the flags needed to enable the indexer plugin. 14 | ```bash 15 | $ ./gen-ghc-wrapper.sh 16 | 17 | USAGE 18 | 19 | ./gen-ghc-wrapper.sh OUTDIR GHC_PATH PLUGIN_DB 20 | 21 | Generates a wrapper for a ghc compiler in OUTDIR that indexes 22 | using the given compiler. 23 | 24 | EXAMPLE 25 | 26 | ./gen-ghc-wrapper.sh ghc_wrapper "$(stack path --compiler-exe)" \ 27 | "$(stack path --snapshot-pkg-db)" 28 | ``` 29 | 30 | ```bash 31 | $ ./gen-ghc-wrapper.sh ghc_wrapper "$(stack path --compiler-exe)" \ 32 | "$(stack path --snapshot-pkg-db)" 33 | ``` 34 | 35 | Now we can index a package and almost all of its dependencies with 36 | ```bash 37 | $ export INDEXER_OUTPUT_DIR=/tmp/indexer-output 38 | $ PATH=$(pwd)/ghc_wrapper:$PATH \ 39 | STACK_ROOT=$HOME/.stack-indexer stack --system-ghc build 40 | ``` 41 | We direct `stack` to use the `ghc` wrapper by including it in the 42 | `PATH` and passing the flag `--system-ghc` to it. 43 | 44 | The purpose of using `STACK_ROOT` above, is to rebuild and index even 45 | the dependencies of `haskell-indexer-plugin`. If you are not 46 | interested in those, then using `STACK_ROOT` can be omitted. 47 | 48 | Packages which use Safe Haskell [can't be indexed currently][sh-issue]. 49 | They can be skipped with 50 | ```bash 51 | $ INDEXER_PLUGIN_ENABLE=0 PATH=$(pwd)/ghc_wrapper:$PATH \ 52 | STACK_ROOT=$HOME/.stack-indexer stack --system-ghc build 53 | ``` 54 | `INDEXER_PLUGIN_ENABLE=0` stops the `ghc` wrapper from using the 55 | plugin, but it stills tells GHC where to find it. The latter is 56 | necessary to workaround [a limitation in GHC][trac15940] when managing 57 | source plugins. 58 | 59 | [trac15940]: https://ghc.haskell.org/trac/ghc/ticket/15940 60 | 61 | To see the result of indexing, all produced files need to be collected 62 | into a serving table. This requires installing [kythe][kythe-install] 63 | in advance. 64 | 65 | Then you can use [serve.sh][serve.sh] to start the web ui. 66 | It picks the indexer output from the environment variable 67 | `INDEXER_OUTPUT_DIR`. 68 | ```bash 69 | $ ../serve.sh localhost:8080 70 | ``` 71 | 72 | [serve.sh]: https://github.com/google/haskell-indexer/blob/master/serve.sh 73 | [kythe-install]: https://github.com/google/haskell-indexer#kythe 74 | 75 | 76 | ### Limitations 77 | 78 | * You shall not index the plugin itself `haskell-indexer-plugin`. 79 | * You shall not index boot libraries. But possibly, they can be 80 | indexed if they are added as packages to the `stack.yaml` file. 81 | * [You shall not index packages which use Safe Haskell][sh-issue]. 82 | 83 | [sh-issue]: https://ghc.haskell.org/trac/ghc/ticket/15920 84 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/src/Language/Haskell/Indexer/Backend/GhcLens.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE GADTs, ScopedTypeVariables, TypeOperators #-} 16 | 17 | -- | Landmine-proof GHC AST traversal. 18 | module Language.Haskell.Indexer.Backend.GhcLens 19 | ( careful 20 | , carefulUniverse 21 | -- * Uniplate-like API (see details). 22 | , universe, children 23 | , universeBi 24 | ) where 25 | 26 | import Control.Applicative 27 | import Control.Exception 28 | import Data.Data 29 | import Data.Hashable 30 | import qualified Data.HashSet as HS 31 | import Data.IORef 32 | import System.IO.Unsafe 33 | import System.Mem.StableName 34 | 35 | import Panic 36 | 37 | data SeenNode where 38 | SeenNode :: StableName a -> SeenNode 39 | 40 | instance Eq SeenNode where 41 | SeenNode sn == SeenNode sn' = eqStableName sn sn' 42 | 43 | instance Hashable SeenNode where 44 | hashWithSalt salt (SeenNode sn) = hashWithSalt salt sn 45 | 46 | type SeenNodes = IORef (HS.HashSet SeenNode) 47 | 48 | {-| 49 | A function that traverses a structure of type @s@ and finds values of 50 | type @a@, applying a function @f@ to lift them into a `Monoid` @m@. 51 | 52 | Does not recurse further into values of type @a@. If it detects a node 53 | it has already traversed, it is skipped using the given Monoid's 'mempty' 54 | in order to prevent traversing self-referential structures forever. 55 | 56 | Nodes that would immediately throw a GHC 'Panic' are also skipped, but other 57 | GHC exceptions are re-thrown. 58 | -} 59 | careful :: (Monoid m, Typeable a, Data s) => (a -> m) -> s -> m 60 | careful f x = unsafeDupablePerformIO $ do 61 | seenNodes <- newIORef HS.empty 62 | carefulChildren seenNodes f x 63 | 64 | -- | Returns all subelements of the same type (including self). 65 | carefulUniverse :: Data a => a -> [a] 66 | carefulUniverse a = a:careful carefulUniverse a 67 | 68 | carefulChildren :: forall m a s. (Monoid m, Typeable a, Data s) 69 | => SeenNodes -> (a -> m) -> s -> IO m 70 | carefulChildren nodesRef f 71 | = gmapQr (liftA2 mappend) (pure mempty) (careful' nodesRef f) 72 | 73 | careful' :: forall m a s. (Monoid m, Typeable a, Data s) 74 | => SeenNodes -> (a -> m) -> s -> IO m 75 | careful' nodesRef f node 76 | = handleGhcException handlePanic $ do 77 | _ <- evaluate node -- check for landmines before we do anything 78 | nodeName <- SeenNode <$> makeStableName node 79 | seenNodes <- readIORef nodesRef 80 | if HS.member nodeName seenNodes 81 | then return mempty 82 | else do 83 | writeIORef nodesRef $! HS.insert nodeName seenNodes 84 | case eqT of 85 | Just (Refl :: s :~: a) -> return (f node) 86 | Nothing -> carefulChildren nodesRef f node 87 | where 88 | handlePanic (Panic _) = return mempty 89 | handlePanic e = throwGhcExceptionIO e 90 | 91 | -- | All values matching the type, deeply from the structure (including root). 92 | universe :: Data a => a -> [a] 93 | universe = carefulUniverse 94 | 95 | -- | All values of type 'a' below the root of type 's'. 96 | -- 97 | -- Warning! Edge case behavior differs from uniplate - if 'a'~'s', uniplate 98 | -- would return only the root, while this returns everything except the root. 99 | -- Such usecase is not intended for universeBi anyway - use universe or 100 | -- children. 101 | universeBi :: forall s a. (Data a, Data s) => s -> [a] 102 | universeBi = concatMap carefulUniverse . (careful return :: s -> [a]) 103 | 104 | -- | Nearest descendants, excluding root. 105 | children :: Data a => a -> [a] 106 | children = careful return 107 | 108 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe/src/Language/Haskell/Indexer/Pipeline/GhcKythe.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE BangPatterns #-} 16 | -- | Basic building block for streamed indexing using GHC backend and Kythe 17 | -- frontend. 18 | module Language.Haskell.Indexer.Pipeline.GhcKythe 19 | ( ghcToKythe 20 | , pluginContinuation 21 | ) where 22 | 23 | import Control.Concurrent.MVar (MVar) 24 | import Control.Monad.Identity (runIdentity) 25 | import Control.Monad.Morph (lift) 26 | import qualified Data.ByteString as B 27 | import Data.Conduit 28 | ( (.|) 29 | , ConduitT 30 | , Void 31 | , await 32 | , awaitForever 33 | , runConduit 34 | , transPipe 35 | , yield 36 | ) 37 | import qualified Data.Text as T 38 | import qualified Data.Text.Encoding as T 39 | import qualified Data.Text.Encoding.Error as T 40 | 41 | import Language.Haskell.Indexer.Translate 42 | import Language.Haskell.Indexer.Backend.AnalysisOptions (AnalysisOptions) 43 | import Language.Haskell.Indexer.Backend.GhcArgs (GhcArgs) 44 | import Language.Haskell.Indexer.Backend.GhcApiSupport (withTypechecked) 45 | import Language.Haskell.Indexer.Frontend.Kythe (toKythe) 46 | 47 | import qualified Language.Kythe.Schema.Raw as Raw 48 | 49 | import System.IO 50 | 51 | ghcToKythe 52 | :: MVar () -> GhcArgs -> AnalysisOptions 53 | -> Raw.VName 54 | -> (Handle -> [Raw.Entry] -> IO ()) 55 | -> IO () 56 | ghcToKythe globalLock ghcArgs analysisOptions baseVName sink = 57 | withTypechecked globalLock ghcArgs analysisOptions (collect (sink stdout) baseVName) 58 | 59 | pluginContinuation :: (AnalysisOptions -> IO XRef) 60 | -> Handle 61 | -> a 62 | -> AnalysisOptions 63 | -> Raw.VName 64 | -> (Handle -> [Raw.Entry] -> IO r) 65 | -> IO () 66 | pluginContinuation action h _ aoes baseVName sink 67 | = action aoes >>= collect (sink h) baseVName 68 | 69 | collect :: ([Raw.Entry] -> IO r) -> Raw.VName -> XRef -> IO () 70 | collect sink baseVName xref = do 71 | sourceText <- lenientDecodeUtf8 . T.unpack 72 | . unSourcePath . analysedTempPath . xrefFile 73 | $ xref 74 | -- Note: since this Conduit pipeline is pretty context-dependent, there 75 | -- is low chance of leak due to accidental sharing (see 76 | -- https://www.well-typed.com/blog/2016/09/sharing-conduit/). 77 | runConduit 78 | $ transPipe (return . runIdentity) 79 | ( toKythe baseVName sourceText xref 80 | -- Batch an ad-hoc number of entries to be emitted together. 81 | .| chunksOf 1000 82 | ) 83 | .| sinkChunks 84 | -- 85 | where 86 | sinkChunks :: ConduitT [Raw.Entry] Void IO () 87 | sinkChunks = awaitForever (lift . sink) 88 | 89 | -- | Haskell sources are de-facto UTF-8, but GHC ignores bad bytes in comments. 90 | -- See http://stackoverflow.com/questions/6797902/haskell-source-encoding. 91 | -- 92 | -- Here we assume that the processed sources already compile with GHC, so 93 | -- lenient decoding suffices. 94 | -- 95 | -- Note: once we start extracting comments with Haddock API, the in-comment 96 | -- offsets might be off if there are incorrect sequences. But also depends on 97 | -- how Haddock behaves, and pretty edge case too. 98 | lenientDecodeUtf8 :: FilePath -> IO T.Text 99 | lenientDecodeUtf8 = fmap (T.decodeUtf8With T.lenientDecode) . B.readFile 100 | 101 | -- | Group a stream into chunks of a given size. The last chunk may contain 102 | -- fewer than n elements. 103 | -- 104 | -- Note: present in upstream Conduit 1.2.9. 105 | chunksOf :: Monad m => Int -> ConduitT a [a] m () 106 | chunksOf n = start 107 | where 108 | start = await >>= maybe (return ()) (\x -> loop n (x:)) 109 | loop !count rest = await >>= maybe (yield (rest [])) go 110 | where 111 | go y | count > 1 = loop (count - 1) (rest . (y:)) 112 | | otherwise = yield (rest []) >> loop n (y:) 113 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/haskell-indexer-backend-ghc.cabal: -------------------------------------------------------------------------------- 1 | name: haskell-indexer-backend-ghc 2 | version: 0.1.0.0 3 | synopsis: Indexing code using GHC API. 4 | description: Part of haskell-indexer, see top-level README.md for more info. 5 | homepage: https://github.com/google/haskell-indexer 6 | license: Apache-2.0 7 | license-file: LICENSE 8 | author: Robin Palotai 9 | maintainer: robinpalotai@google.com 10 | copyright: Google Inc. 11 | category: Language 12 | build-type: Simple 13 | cabal-version: 2.0 14 | extra-source-files: testdata/basic/*.hs 15 | , testdata/basic/*.c 16 | , testdata/typelink/*.hs 17 | 18 | library 19 | hs-source-dirs: src 20 | exposed-modules: Language.Haskell.Indexer.Backend.Ghc 21 | Language.Haskell.Indexer.Backend.GhcLens 22 | Language.Haskell.Indexer.Backend.GhcArgs 23 | Language.Haskell.Indexer.Backend.GhcApiSupport 24 | Language.Haskell.Indexer.Backend.GhcEnv 25 | build-depends: Cabal >= 2.4 && < 3.3 26 | , base >=4.8 && <5 27 | , bytestring 28 | , containers 29 | , directory 30 | , filepath 31 | , ghc 32 | , ghc-paths 33 | , hashable 34 | , haskell-indexer-backend-core >= 0.1 35 | , haskell-indexer-translate >= 0.1 36 | , mtl 37 | , network-uri >= 2.6 && < 3 38 | , reflection >= 1.2 39 | , text 40 | , transformers 41 | , unix 42 | , unordered-containers 43 | ghc-options: -Wall 44 | -Wcompat 45 | -Wincomplete-record-updates 46 | -Wincomplete-uni-patterns 47 | -Wredundant-constraints 48 | default-language: Haskell2010 49 | 50 | Test-Suite basic_test 51 | default-language: Haskell2010 52 | type: exitcode-stdio-1.0 53 | main-is: BasicTest.hs 54 | hs-source-dirs: tests 55 | other-modules: Language.Haskell.Indexer.Backend.Ghc.Test.TestShim 56 | Language.Haskell.Indexer.Backend.Ghc.Test.BasicTestBase 57 | Language.Haskell.Indexer.Backend.Ghc.Test.TranslateAssert 58 | Language.Haskell.Indexer.Backend.Ghc.Test.TestHelper 59 | build-depends: base 60 | , bytestring 61 | , directory 62 | , filepath 63 | , ghc 64 | , haskell-indexer-backend-core >= 0.1 65 | , haskell-indexer-backend-ghc 66 | , haskell-indexer-translate >= 0.1 67 | , network-uri >= 2.6 && < 3 68 | , semigroups 69 | , temporary 70 | , test-framework 71 | , test-framework-hunit 72 | , text 73 | , text-offset 74 | , transformers 75 | , HUnit 76 | ghc-options: -Wall 77 | -Wcompat 78 | -Wincomplete-record-updates 79 | -Wincomplete-uni-patterns 80 | -Wredundant-constraints 81 | 82 | Test-Suite typelink_test 83 | default-language: Haskell2010 84 | type: exitcode-stdio-1.0 85 | main-is: TypeLinkTest.hs 86 | hs-source-dirs: tests 87 | other-modules: Language.Haskell.Indexer.Backend.Ghc.Test.TypeLinkTestBase 88 | Language.Haskell.Indexer.Backend.Ghc.Test.TestShim 89 | Language.Haskell.Indexer.Backend.Ghc.Test.TranslateAssert 90 | Language.Haskell.Indexer.Backend.Ghc.Test.TestHelper 91 | build-depends: base 92 | , bytestring 93 | , directory 94 | , filepath 95 | , ghc 96 | , haskell-indexer-backend-core >= 0.1 97 | , haskell-indexer-backend-ghc 98 | , haskell-indexer-translate >= 0.1 99 | , network-uri >= 2.6 && < 3 100 | , semigroups 101 | , temporary 102 | , test-framework 103 | , test-framework-hunit 104 | , text 105 | , text-offset 106 | , transformers 107 | , HUnit 108 | ghc-options: -Wall 109 | -Wcompat 110 | -Wincomplete-record-updates 111 | -Wincomplete-uni-patterns 112 | -Wredundant-constraints 113 | -------------------------------------------------------------------------------- /text-offset/tests/OffsetTest.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE OverloadedStrings #-} 16 | module Main where 17 | 18 | import qualified Data.ByteString.Lazy as BL 19 | import qualified Data.Text.Lazy as TL 20 | import qualified Data.Text.Lazy.Encoding as TL 21 | 22 | import Test.Framework (defaultMain) 23 | import Test.Framework.Providers.HUnit (testCase) 24 | import Test.Framework.Providers.QuickCheck2 (testProperty) 25 | import Test.HUnit 26 | import Test.QuickCheck 27 | 28 | import Data.Char (chr) 29 | import Data.Maybe (fromJust) 30 | import Data.Text.Offset 31 | 32 | main :: IO () 33 | main = defaultMain 34 | [ testProperty "bytecount matches final offset" 35 | propBytecountMatchesFinalOffset 36 | , testCase "column over end of line" testColOverEndOfLine 37 | , testCase "column over end of file" testColOverEndOfFile 38 | , testCase "line over end of file" testLineOverEndOfFile 39 | , testCase "multibytes" testMultiBytes 40 | ] 41 | 42 | -- | String with more multibytes (Arbitrary for String gives you at most 43 | -- 2-bytes). 44 | newtype MultiString = MultiString { unMultiString :: String } 45 | deriving (Eq, Show) 46 | 47 | instance Arbitrary MultiString where 48 | arbitrary = MultiString `fmap` listOf genChar 49 | where 50 | genChar :: Gen Char 51 | genChar = fmap chr . frequency $ 52 | [ (8, choose(0, 0x7f)) 53 | , (1, choose(0x80, 0x7ff)) 54 | , (1, choose(0x800, 0xffff)) 55 | , (1, choose(0x10000, 0x10fff)) -- subset of remaining range. 56 | ] 57 | 58 | propBytecountMatchesFinalOffset :: [MultiString] -> Property 59 | propBytecountMatchesFinalOffset mss = 60 | let ts = map (TL.pack . unMultiString) mss 61 | -- | Getting back the last valid source offset is not straightforward, 62 | -- since we don't consider newlines valid source positions. So rather 63 | -- just make user we don't have a terminal newline, and cover that case 64 | -- in other tests. 65 | t = TL.intercalate "\n" ts `TL.append` "guard" 66 | table = createOffsetTable t 67 | tlines = TL.splitOn "\n" t 68 | queryLine = length tlines - 1 69 | queryCol = (fromIntegral . TL.length . last $ tlines) - 1 70 | afterLastByteOffset = 71 | fromJust $ (+1) `fmap` lineColToByteOffset table queryLine queryCol 72 | encodedLength = fromIntegral . BL.length . TL.encodeUtf8 $ t 73 | tl = fromIntegral . TL.length $ t 74 | in classify (length tlines == 1) "single-line" $ 75 | classify (length tlines > 1) "multi-line" $ 76 | classify (encodedLength > tl) "multibyte" $ 77 | classify (encodedLength == tl) "singlebyte" $ 78 | afterLastByteOffset === encodedLength 79 | 80 | testColOverEndOfLine :: Assertion 81 | testColOverEndOfLine = do 82 | let t = createOffsetTable "012\n345" 83 | assertEqual "valid offset" (Just 2) (lineColToByteOffset t 0 2) 84 | assertEqual "offset at end" (Left (OverLineEnd 3 JustAtLineEnd)) 85 | (lineColToByteOffsetDetail t 0 3) 86 | assertEqual "offset after end" (Left (OverLineEnd 4 AfterLineEnd)) 87 | (lineColToByteOffsetDetail t 0 4) 88 | 89 | testColOverEndOfFile :: Assertion 90 | testColOverEndOfFile = do 91 | let t = createOffsetTable "012\n345" 92 | assertEqual "valid offset before EOF" (Just 6) 93 | (lineColToByteOffset t 1 2) 94 | assertEqual "invalid offset over EOF" (Left (OverLineEnd 7 JustAtLineEnd)) 95 | (lineColToByteOffsetDetail t 1 3) 96 | 97 | testLineOverEndOfFile :: Assertion 98 | testLineOverEndOfFile = do 99 | assertEqual "single line, no newline" (Left NoSuchLine) 100 | (lineColToByteOffsetDetail (createOffsetTable "012") 1 0) 101 | assertEqual "single line, with newline" (Left EmptyLine) 102 | (lineColToByteOffsetDetail (createOffsetTable "012\n") 1 0) 103 | assertEqual "negative test: non-empty newline" (Just 4) 104 | (lineColToByteOffset (createOffsetTable "012\n0") 1 0) 105 | 106 | testMultiBytes :: Assertion 107 | testMultiBytes = do 108 | let t = createOffsetTable "0む4\nΣΣ8" 109 | assertEqual "before 3-byte" (Just 1) (lineColToByteOffset t 0 1) 110 | assertEqual "after 3-byte" (Just 4) (lineColToByteOffset t 0 2) 111 | assertEqual "after 2x2-byte last line" (Just 10) 112 | (lineColToByteOffset t 1 2) 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | The haskell-indexer package provides libs for preprocessing Haskell source code 2 | into a representation for easy entity cross-referencing, as well as a frontend 3 | for emitting entities for the [Kythe](https://kythe.io) indexing schema. 4 | 5 | This is not an official Google product. 6 | 7 | # Supported systems 8 | 9 | Indexing hosts: 10 | - Linux: supported - follow below documentation. 11 | - Windows, MacOS: didn't try - backend part likely compiles, wiring and Kythe 12 | frontend likely not (see #38). 13 | 14 | Compilers: 15 | - GHC 8.6.5 16 | - GHC 8.8.1 (planned) 17 | 18 | Stackage: 19 | - A recent LTS release corresponding to above compilers is supported. 20 | See `stack-ghcXXX.yml` files. 21 | 22 | Previous compilers were supported at some point. Checkout an old repository 23 | state if interested: 24 | - [GHC 7.10.3](https://github.com/google/haskell-indexer/tree/last-ghc-7.10.3) 25 | - [GHC 8.4](https://github.com/google/haskell-indexer/tree/last-ghc-8.4) 26 | 27 | [![Build Status](https://travis-ci.org/google/haskell-indexer.svg?branch=master)](https://travis-ci.org/google/haskell-indexer) 28 | 29 | # Installation 30 | 31 | ## Stack 32 | 33 | Download Stack from http://docs.haskellstack.org 34 | 35 | ## Kythe 36 | 37 | If you want to use the Kythe frontend, you'll need to install it either from 38 | source or from the official release. The latter is easier, but the web UI has 39 | been removed in recent versions. 40 | 41 | ### Official Release 42 | 43 | Download a [Kythe release](https://github.com/google/kythe/releases) and unpack 44 | it. 45 | 46 | ``` 47 | tar xzf kythe-v0.0.26.tar.gz -C /opt/ 48 | rm -r /opt/kythe 49 | ln -s /opt/kythe-v0.0.26 /opt/kythe 50 | chmod -R 755 /opt/kythe/web/ui # It misses permission by default. 51 | ``` 52 | 53 | Version `v0.0.30` is the latest version that includes the web UI. If you want a 54 | newer Kythe than this, you'll need to build from source. 55 | 56 | If you want to install Kythe in a different location to `/opt/kythe` then you 57 | should also set `KYTHE_DIR` to the location of the installation. 58 | 59 | ### Building From Source 60 | 61 | Clone Kythe from [its GitHub repo](https://github.com/google/kythe/releases) and 62 | follow the [Getting Started guide](https://kythe.io/getting-started/) to build 63 | and install it into `/opt/kythe`. Then, from within the Kythe clone, build the 64 | web frontend and copy its files into their rightful place: 65 | 66 | ``` 67 | bazel build //kythe/web/ui 68 | mkdir -p /opt/kythe/web/ui 69 | cp -r bazel-bin/kythe/web/ui/resources/public/* /opt/kythe/web/ui 70 | cp -r kythe/web/ui/resources/public/* /opt/kythe/web/ui 71 | chmod -R 755 /opt/kythe/web/ui 72 | ``` 73 | 74 | ## Protoc 3 75 | 76 | Download the latest [Proto compiler 3 77 | release](https://github.com/google/protobuf/releases), unpack it and place the 78 | binary in the PATH. 79 | 80 | ``` 81 | unzip -j protoc-*-linux-x86_64.zip bin/protoc -d /usr/local/bin/ 82 | ``` 83 | 84 | > If you use have Nix installed and you use `stack --nix`, you do not need to do 85 | > this. 86 | 87 | ## Haskell Indexer Plugin (ghc >= 8.6 only) 88 | 89 | Haskell modules can be indexed with a GHC source plugin while building a 90 | project. Whatever build system is in use, indexing can be achieved by 91 | ensuring that the invocations to `ghc` include the flags that enable the 92 | plugin. 93 | 94 | For instruction on how to install and use the plugin with `stack`, see 95 | [stack-example/README.md](stack-example/README.md). 96 | 97 | If you are using some other build system, the following GHC options are 98 | relevant after the plugin is installed. 99 | 100 | * `-package-db `: Tells the package database where the plugin has 101 | been installed. It may be used more than once if the plugin dependencies 102 | spread through more than one package database. 103 | * `-plugin-package haskell-indexer-plugin`: Tells ghc to expose the package 104 | containing the plugin, so it can be found when needed. 105 | * `-fplugin Haskell.Indexer.Plugin`: Tells to use the plugin when compiling 106 | modules. 107 | * `-fplugin-opt Haskell.Indexer.Plugin:-o` and 108 | `-fplugin-opt Haskell.Indexer.Plugin:`: Tell the plugin where 109 | to place the output of indexing. 110 | 111 | # Build the project 112 | 113 | Use the following to build and run tests: 114 | 115 | ``` 116 | git clone --recursive https://github.com/google/haskell-indexer.git 117 | cd haskell-indexer 118 | export STACK_YAML=$(readlink -f stack-ghc865.yaml) 119 | stack build && stack test 120 | # To test Kythe frontend: 121 | pushd kythe-verification; stack install && ./test.sh; popd 122 | ``` 123 | 124 | To test all supported stack configurations, do `./run-ghc-tests.sh`. 125 | 126 | # Demo 127 | 128 | To index a few packages, run: 129 | 130 | ```bash 131 | export INDEXER_OUTPUT_DIR=/tmp/indexer-output 132 | ./build-stack.sh mtlparse cpu 133 | ``` 134 | 135 | The script adds a wrapper for the GHC compiler used by Stack (`stack path --compiler-exe`), does the indexing when `ghc --make` is specified on the command line to build a package. You can run `build-stack.sh` multiple times. 136 | 137 | To serve the index at `http://localhost:8080`: 138 | 139 | ```bash 140 | ./serve.sh localhost:8080 141 | ``` 142 | 143 | If you get empty index, look at `$INDEXER_OUTPUT_DIR/*.stderr` files about 144 | possible indexing errors. Also, make sure that the `*.entries` files are not 145 | empty. If they are, it indicates that `ghc_kythe_wrapper` failed to index. 146 | 147 | ## Indexing using Docker 148 | 149 | If you plan to use the Dockerized build feature of stack, please install 150 | Docker. It is also advised to set up a docker wrapper script by following the 151 | instructions at the [stack Docker 152 | security](https://docs.haskellstack.org/en/stable/docker_integration/#security) 153 | section. 154 | 155 | The docker image has all C library dependencies so it's possible to use it to 156 | index the whole Stackage snapshot. See `stack-build-docker.sh` for a 157 | comprehensive example of indexing a Stackage snapshot, and serving a Kythe 158 | index. 159 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | Contains subheaders about larger improvement efforts. 4 | 5 | ## `rules_haskell` support 6 | 7 | Context: [`rules_haskell`](https://github.com/tweag/rules_haskell) supports 8 | building Haskell targets by extending Bazel, openly maintained by Tweag. 9 | It would be useful if `haskell-indexer` supported generating crossreferences for 10 | `rules_haskell` out of the box. 11 | 12 | ### Goals 13 | 14 | Provide crossreference (as Kythe entries) for both `rules_haskell`-managed 15 | and `nixpkgs`-sourced compilations. Latter is used by `rules_haskell` for hackage 16 | imports, instead of hand-rolled source imports. 17 | 18 | Crossreference data should be serveable through Kythe's language server, ready to 19 | be consumed by emacs's lsp-mode (with modifications if needed). 20 | 21 | Bonus goal: based on local edits, either make partial index updates possible, or 22 | merge local updates with the static backing index at serving time. 23 | 24 | Possible future expansions: 25 | * xref `proto-lens` generated Haskell protobuf code with proto sources. 26 | (https://github.com/google/haskell-indexer/issues/15) 27 | * xref Haskell FFI code with C code 28 | (https://github.com/google/haskell-indexer/issues/18) 29 | 30 | #### Rationale 31 | 32 | In the non-Bazel world, one might use 33 | [haskell-ide-engine](https://github.com/haskell/haskell-ide-engine) as a 34 | [LSP](https://microsoft.github.io/language-server-protocol/) server over a 35 | stack/cabal workspace, providing editor support (such as type sig requests, 36 | code navigation, completion). 37 | 38 | For Bazel/rules_haskell, one can either hand-roll the stack/cabal files in 39 | addition to the BUILD files to run the server. But that is duplicate work, also 40 | there are scaling concerns on large workspaces. 41 | 42 | So one way to get LSP support for rules_haskell is to get the Kythe entries 43 | using haskell-indexer, and serve them through [Kythe's language 44 | server](https://github.com/kythe/kythe/tree/master/kythe/go/languageserver). 45 | 46 | An alternative would be using GHC's HIE files (expected to land in GHC 8.8) to 47 | write an [LSIF](https://github.com/Microsoft/language-server-protocol/issues/623) index directly, which could be served by `haskell-ide-engine` 48 | without much overhead. This approach is not pursued here, but might be revisited 49 | in the future, especially if LSIF demonstrates a solid handling of cross-project 50 | / cross-language references (see https://gist.github.com/robinp/76f9d3d91387da5162f773895d4e1d15). 51 | 52 | ### Tasks 53 | 54 | * Decide how compilations will be extracted/analysed. 55 | 56 | * For quick productivity boost, run analysis directly, without extraction. 57 | * `rules_haskell`: maybe just indicate indexing by passing some global flag 58 | (if possible with Bazel). Alternatively use `aspects`. 59 | * `nixpkgs`: plumb a `should-index` flag to whatever helper is used to define 60 | a Haskell package. Make `rules_haskell` pass this flag. 61 | * Kythe entries will be written to a directory defined by the 62 | `KYTHE_OUTPUT_DIRECTORY` env (convention). 63 | 64 | * Sync with Kythe/Bazel about merits of extraction, vs direct analysis. 65 | 66 | * TLDR extraction helps distributing the analysis workload (analysing machines 67 | are different than those preparing the build), but is an overhead until that 68 | scale is hit (personal idea, to be verified). 69 | 70 | * If extraction is needed, decide between `action_listener` vs `aspects`. 71 | * TLDR: action listeners are older mechanism, for providing info about how 72 | the compiler was invoked to build a given target. Needs heuristics and/or 73 | parsing the compiler command-line for non-canonical languages (like 74 | Haskell). Aspects are newer, more general, and able to retain precise 75 | info, so less parsing is needed. Should also let accurate access to the 76 | build chain, leaving less guesswork to the extractor about which inputs 77 | should be captured. 78 | 79 | * Enhancing `haskell-indexer` deficits: 80 | 81 | * Output `MarkedSource` protos, so function signatures can be served by Kythe. 82 | (https://github.com/google/haskell-indexer/issues/4) 83 | 84 | * Note: would be nice to keep the intermediate Type representation isomorphic 85 | to HIE's HieType, so switching haskell-indexer to use HIE as source later 86 | is smooth (also to avoid duplicate work). 87 | 88 | * Optional: export types to Kythe. This is tricky since Haskell's types are 89 | complex, and Kythe's schema can't represent them without some loss. Would 90 | propose to defer this until missing type crosslinking actually hurts 91 | navigation experience. (https://github.com/google/haskell-indexer/issues/31) 92 | 93 | * Note: Kythe team is open to schema suggestions, but we need a good usecase 94 | first before engineering a lot (is there any static analysis that would 95 | benefit from fine-grained Kythe-level types?) 96 | 97 | * Any other `haskell-indexer` improvements as well, to increase Haskell feature 98 | coverage. 99 | 100 | * Note: in the advent of [HIE](https://ghc.haskell.org/trac/ghc/wiki/HIEFiles), 101 | we shouldn't enhance Haskell source features, but rather wait to switch to 102 | HIE, so we can get rid of our custom GHC API-based mining. What we should 103 | concentrate on are rather cross-language features (that need effort regardless 104 | of HIE). 105 | 106 | * Serving xrefs from local updates: 107 | * Research/discuss with Kythe team about plans and support, if any. 108 | * Prototype. Pretty open-ended at this point, to be updated. 109 | 110 | ## Glossary and links 111 | 112 | * Bazel action listener: gets ExtraActionInfo proto for Bazel targets, can 113 | identify inputs and compiler command based on them. 114 | https://docs.bazel.build/versions/master/be/extra-actions.html 115 | 116 | * Bazel aspect: a shadow build graph with custom actions. 117 | https://docs.bazel.build/versions/master/skylark/aspects.html 118 | 119 | * Kythe: schema for language agnostic, approximate AST, also tools for serving 120 | xref quries based on this data. 121 | 122 | * HIE: Haskell Interface Extended files, containing a simplified AST, with 123 | scopes and (constrained) types already resolved. Likely suitable to use as a 124 | haskell-indexer backend. See https://ghc.haskell.org/trac/ghc/wiki/HIEFiles 125 | and https://github.com/ghc/ghc/tree/master/compiler/hieFile. 126 | 127 | -------------------------------------------------------------------------------- /stack-ghc-8.10.yaml: -------------------------------------------------------------------------------- 1 | resolver: ghc-8.10.1 2 | 3 | packages: 4 | - haskell-indexer-backend-core 5 | - haskell-indexer-backend-ghc 6 | - haskell-indexer-frontend-kythe 7 | - haskell-indexer-pathutil 8 | - haskell-indexer-pipeline-ghckythe 9 | - haskell-indexer-pipeline-ghckythe-wrapper 10 | - haskell-indexer-translate 11 | - kythe-proto 12 | - kythe-schema 13 | - text-offset 14 | 15 | extra-deps: 16 | - HUnit-1.6.0.0@sha256:8014b27c884becd8041214a1ab263fa92244ca62700a911aff604a1047869298,1570 17 | - QuickCheck-2.13.2@sha256:636e7265bf75122e7e2f97627c47aad3b772ee3b35b134cafb6095116ce8d07a,6974 18 | - StateVar-1.2@sha256:9ab3e4a0e252d28bc2f799c83e0725c3e23e8d3b722cff0fdb9822e64b6c16ac,1413 19 | - ansi-terminal-0.10.3@sha256:e2fbcef5f980dc234c7ad8e2fa433b0e8109132c9e643bc40ea5608cd5697797,3226 20 | - ansi-wl-pprint-0.6.9@sha256:f6fd6dbd4adcad0432bf75e5f5b19bb1deda00a1d8056faf18090026e577652d,2388 21 | - base-orphans-0.8.2@sha256:40ef37ed043aac2cbb6c538fdebfc62e601ee65ee161e4a6327452133b574d7e,2958 22 | - bifunctors-5.5.7@sha256:54c38cffb60213bfbb9474d92226696c30d07bc6f88e1e9c887ab7c5a5721b45,3410 23 | - cabal-doctest-1.0.8@sha256:34dff6369d417df2699af4e15f06bc181d495eca9c51efde173deae2053c197c,1491 24 | - call-stack-0.2.0@sha256:5ce796b78d5f964468ec6fe0717b4e7d0430817f37370c47b3e6b38e345b6643,1202 25 | - colour-2.3.5@sha256:b27db0a3ad40d70bdbd8510a104269f8707592e80757a1abc66a22ba25e5a42f,1801 26 | - comonad-5.0.6@sha256:65117d3d9e75ded548d2bbbd5ca9f5e6e1e42ecee08175aa6a77666753f797e3,3348 27 | - conduit-1.3.2@sha256:f85b5dc1d7d30e9122ac15975e7f59fe846b42ef952e2beddd32520a753c2fc4,4980 28 | - contravariant-1.5.2@sha256:853259271870000c007a281f0bf0bf6e1aaa97c5fd5cd5734d7b0d79b9de2af5,2761 29 | - data-default-0.7.1.1@sha256:2804e8d14f521a1edee88b68b66347448e7f3b685868290fdc55930e4471f5a9,645 30 | - data-default-class-0.1.2.0@sha256:63e62120b7efd733a5a17cf59ceb43268e9a929c748127172d7d42f4a336e327,542 31 | - data-default-instances-containers-0.0.1@sha256:6e1f4b28028a3bc455aaf4b5a9104b71ea72cff78b1b8041863df7afd1a8deb3,527 32 | - data-default-instances-dlist-0.0.1@sha256:4286abacbb256c392907701be16986a6e07f2beaf2778e7bd925465655d9e301,507 33 | - data-default-instances-old-locale-0.0.1@sha256:d4a757f68f0f83531fcb34a4525fe6769c54a45182e28ffdfff19c2b0ace42fb,526 34 | - distributive-0.6.2@sha256:6e97ccea6b887eecd74896dcc96cece54a0c5ff4848a977c1db81b8a67bc7aea,3062 35 | - dlist-0.8.0.8@sha256:90ca348bffdc62d7070bcf0e97c728f8d01b24fbc7ea08d2909157d0da76534c,2066 36 | - extensible-exceptions-0.1.1.4@sha256:eb5fe684a7ffe8d1ed2ed6cdaec7dfb29efc780811ea7158a64edc2abc516f47,1205 37 | - ghc-paths-0.1.0.12@sha256:85370fdc615d4be5e09d9269eebb9a3fc7017c40b1a9e0050b121d75908564bd,632 38 | - ghc-source-gen-0.4.0.0@sha256:b6cdde6b183e70f6fbed3b0a45ad6640204b636e9a7fb898c5d7ec79719f14a1,3698 39 | - hashable-1.3.0.0@sha256:4c70f1407881059e93550d3742191254296b2737b793a742bd901348fb3e1fb1,5206 40 | - hostname-1.0@sha256:8203b6ecd14ca1ef12f73a471b0a6a4be3ad4568d8b84f2bc4bc9e0abb8c4153,813 41 | - lens-family-2.1.0@sha256:de40e0281e3410a4a742b748dd4054f3ff6694c2398638da6b95f593b400aa1e,1989 42 | - lens-family-core-2.1.0@sha256:124b4371ffc147a8f2fb59a7bda339ce20eec83f887a647a63497e1efc56f837,2290 43 | - mmorph-1.1.3@sha256:abfc95648fef0008f984b94137ce8e1635fb071c7bfaaa7393ba175a1b3bb12f,919 44 | - mono-traversable-1.0.15.1@sha256:cad0e8681cd6c96d3303867fc68c80e2f5d55c2c4bf5277c06ca74402fda61c8,2230 45 | - network-uri-2.6.3.0@sha256:d2d9ff3a80d9b2d1ff317a354bc0c56cc109c69a4c2449e5fc712d3ddce83ede,2902 46 | - old-locale-1.0.0.7@sha256:fa998be2c7e00cd26a6e9075bea790caaf3932caa3e9497ad69bc20380dd6911,1071 47 | - optparse-applicative-0.15.1.0@sha256:29ff6146aabf54d46c4c8788e8d1eadaea27c94f6d360c690c5f6c93dac4b07e,4810 48 | - primitive-0.7.0.1@sha256:a381571c36edc7dca28b77fe8159b43c14c640087ec5946adacf949feec64231,3433 49 | - profunctors-5.4@sha256:545fdbc05131fa29e6612e915ec5d4dadfbcf3a6def86c8b95ca26593b21b259,2073 50 | - proto-lens-0.7.0.0@sha256:2c44a62375f7712f9381f84b1d30cee2f94384f1c98801db2f4450359a8e5036,3036 51 | - proto-lens-protoc-0.7.0.0@sha256:04e57ec59ce21dc781b9debe063f03d85a7ec2ad1ee5c2deb1c30282119e9119,2304 52 | - proto-lens-runtime-0.7.0.0@sha256:d6cfab159a63f5c42a1a507638c674a714dfa5f69a06f559e8da840eaafde3ab,3115 53 | - proto-lens-setup-0.4.0.4@sha256:c44d1b6d68a8faf1d2f9c7ca492ce3c9c2ee66d255452c683f762f10c9ebe430,3185 54 | - random-1.1@sha256:7b67624fd76ddf97c206de0801dc7e888097e9d572974be9b9ea6551d76965df,1777 55 | - reflection-2.1.6@sha256:044acb7caf41a9d8246878f849baed2dffbc4582d0a1e5c7c079d4287239e970,3976 56 | - regex-base-0.94.0.0@sha256:44aa95ca762294ffbb28cf0af9c567d93b5d2c56e4f38ce5385a257d899f968e,2253 57 | - regex-posix-0.96.0.0@sha256:690d1366e9ea6df71ded3daf49ca3d53c63eda3f95937962299b2391824dc3b3,2531 58 | - resourcet-1.2.4@sha256:1dbd6b3fdd91ab693e7d0d7389c1c482b3fef3df18b552bd23da8e16018f5047,1655 59 | - semigroups-0.19.1@sha256:ecae129621e0d2f77bef2f01e4458c2e0567ab6e1f39579c61d7cec8058ebb0e,6262 60 | - split-0.2.3.4@sha256:bad9bfc9e8e0aca4ac1f7b3a2767ee6b1c199d4a1b6960db5cc3d7674aee5d73,2564 61 | - splitmix-0.0.5@sha256:4a49661be63f5aea0e132e9ab51be918789bb4dceb4ab9b15b85ba9cbbef5999,5412 62 | - tagged-0.8.6@sha256:1f7ca84e6c88cbb923641c60041c9f56c34f1a889759cc073cdf10542b441ff9,2606 63 | - temporary-1.3@sha256:3a66c136f700dbf42f3c5000ca93e80b26dead51e54322c83272b236c1ec8ef1,1485 64 | - test-framework-0.8.2.0@sha256:065a1a6f5d43cee0c01c9d7f1a9539c62661a91012c6314db31fbed1a512e9a1,6395 65 | - test-framework-hunit-0.3.0.2@sha256:7fd007e9cb082cd64a2213a6d36acf057f7d6df6b5343a088e81b2b3a9a23545,1487 66 | - test-framework-quickcheck2-0.3.0.5@sha256:c25bfe518324a584334baa8139491624d72ee0e04942692a384058b54372efaa,1840 67 | - th-abstraction-0.3.2.0@sha256:9b5b4e6e2bbff9b075ad7751ee98e2107090bd17b51d5442695b8990e4db6521,1851 68 | - transformers-compat-0.6.5@sha256:50b00c57bf3fc379ec2477bfc261a2aebc983084488478adb29854f193af4696,5490 69 | - unliftio-core-0.2.0.1@sha256:9b3e44ea9aacacbfc35b3b54015af450091916ac3618a41868ebf6546977659a,1082 70 | - unordered-containers-0.2.10.0@sha256:5e9b095a9283d9e2f064fec73a81a6b6ea0b7fda3f219a8175785d2d2a3de204,5199 71 | - vector-0.12.1.2@sha256:9291bc581f36e51d5bda9fce57cb980fbec3dd292996896f285fef39eb80a9a0,7364 72 | - vector-algorithms-0.8.0.3@sha256:477ef5ac82fdd28a39536ed0a767faec3425802827abee485be31db5bc6f5e90,3488 73 | - void-0.7.3@sha256:13d30f62fcdf065e595d679d4ac8b4b0c1bb1a1b73db7b5b5a8f857cb5c8a546,1857 74 | - xml-1.3.14@sha256:edb7cf08e80013cab3f80e1cb1765cb8e4f0772cf5de8a50ba3bbdfa2ae61d8c,942 75 | 76 | # Allow our custom Setup.hs scripts to import Data.ProtoLens.Setup from the version of 77 | # `proto-lens-protoc` in stack's local DB. See: 78 | # https://github.com/google/proto-lens/blob/master/README.md#using-cabal 79 | explicit-setup-deps: 80 | "*": true 81 | 82 | nix: 83 | packages: [gcc, protobuf] 84 | 85 | allow-newer: true 86 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/tests/Language/Haskell/Indexer/Backend/Ghc/Test/TestHelper.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {- Helpers for testing extraction functionality. 16 | - 17 | - For now hardcoded for GHC backend. If multiple backends would be available, 18 | - should make it injectable. 19 | -} 20 | {-# LANGUAGE OverloadedStrings #-} 21 | module Language.Haskell.Indexer.Backend.Ghc.Test.TestHelper 22 | ( TestEnv(..) 23 | , analyse 24 | , mergeRefs 25 | -- * Combined helpers. 26 | , assertXRefsFrom 27 | , assertXRefsFromExtra 28 | -- * Reexports from TranslateAssert. 29 | , module Language.Haskell.Indexer.Backend.Ghc.Test.TranslateAssert 30 | ) where 31 | 32 | import Control.Concurrent.MVar (MVar, newMVar) 33 | import Control.Monad.Trans.Class (lift) 34 | import Control.Monad.Trans.Reader (ReaderT, asks, runReaderT) 35 | import Control.Monad.IO.Class (liftIO) 36 | import Data.Either (lefts) 37 | import qualified Data.Foldable as Foldable 38 | import qualified Data.IORef as IO 39 | import Data.List.NonEmpty (NonEmpty) 40 | import qualified Data.List.NonEmpty as NonEmpty 41 | import qualified System.Directory as Dir 42 | import System.FilePath (()) 43 | import qualified System.FilePath as File 44 | import System.IO.Temp (withTempDirectory) 45 | import System.IO.Unsafe (unsafePerformIO) 46 | 47 | import Language.Haskell.Indexer.Backend.GhcArgs 48 | import Language.Haskell.Indexer.Backend.AnalysisOptions 49 | import Language.Haskell.Indexer.Backend.GhcApiSupport (withTypechecked) 50 | import Language.Haskell.Indexer.Backend.Ghc.Test.TranslateAssert -- to reexport 51 | 52 | -- | A sin against humanity in the name of easier testing. 53 | -- This is the evil but de-facto global variable pattern. 54 | globalLock :: MVar () 55 | globalLock = unsafePerformIO (newMVar ()) 56 | {-# NOINLINE globalLock #-} 57 | 58 | data TestEnv = TestEnv 59 | { testDataDirectory :: FilePath 60 | -- ^ Path up to the data directory containing subdirs per test. 61 | , testDefaultGhcArgs :: GhcArgs 62 | -- ^ Defaults populated with locations of various tools. 63 | } 64 | 65 | -- | Like 'assertXRefsFromExtra' but without extra files to copy. 66 | assertXRefsFrom :: [String] -> ReaderT XRef IO () -> ReaderT TestEnv IO () 67 | assertXRefsFrom = assertXRefsFromExtra [] 68 | 69 | -- | Convenience for asserting analysis of testdata-relative files and other 70 | -- arguments. The 'extraFiles' get added to the input directory, but don't get 71 | -- passed on the GHC command line. 72 | assertXRefsFromExtra 73 | :: [FilePath] -> [String] -> ReaderT XRef IO () 74 | -> ReaderT TestEnv IO () 75 | assertXRefsFromExtra extraFiles args asserts = do 76 | xref <- fmap mergeRefs (analyse extraFiles args) 77 | liftIO (runReaderT asserts xref) 78 | 79 | -- | Analyses invocation with given args, relative to the 'data' directory. 80 | analyse :: [String] -> [String] -> ReaderT TestEnv IO (NonEmpty XRef) 81 | analyse extraFiles relativeArgs = do 82 | dataDir <- asks testDataDirectory 83 | let convertToFullPath = mapM (makeFullPath dataDir) 84 | args <- liftIO $ convertToFullPath relativeArgs 85 | extraFullPaths <- fmap lefts . liftIO $ convertToFullPath extraFiles 86 | analyiseFullPath extraFullPaths args 87 | where 88 | makeFullPath :: FilePath -> String -> IO Arg 89 | makeFullPath baseDir arg = do 90 | let candidatePath = baseDir arg 91 | exists <- Dir.doesFileExist candidatePath 92 | return $ if not exists then Right arg -- Not a file arugment. 93 | else Left candidatePath 94 | 95 | -- | We can either pass (here absolute) filenames or free-form strings as GHC 96 | -- options. 97 | type Arg = Either FilePath String 98 | 99 | -- | Merges with terrible list concat performance, keeps an arbitrary file 100 | -- name. 101 | mergeRefs :: NonEmpty XRef -> XRef 102 | mergeRefs = Foldable.foldr1 merge 103 | where 104 | merge a b = XRef 105 | { xrefFile = xrefFile a 106 | , xrefModule = xrefModule a 107 | , xrefDecls = xrefDecls a ++ xrefDecls b 108 | , xrefDocDecls = xrefDocDecls a ++ xrefDocDecls b 109 | , xrefModuleDocDecls = xrefModuleDocDecls a ++ xrefModuleDocDecls b 110 | , xrefCrossRefs = xrefCrossRefs a ++ xrefCrossRefs b 111 | , xrefRelations = xrefRelations a ++ xrefRelations b 112 | , xrefImports = xrefImports a ++ xrefImports b 113 | } 114 | 115 | analyiseFullPath :: [FilePath] -> [Arg] -> ReaderT TestEnv IO (NonEmpty XRef) 116 | analyiseFullPath extraPaths args = do 117 | tmpTop <- liftIO Dir.getTemporaryDirectory 118 | -- Put resources of each test into separate tmp dir to prevent crosstalk. 119 | withTempDirectory tmpTop "ghctest." $ \tmp -> do 120 | mirrors <- liftIO $ do 121 | mapM_ (makeMirror tmp) extraPaths 122 | mapM (either (makeMirror tmp) return) args 123 | baseGhcArgs <- asks testDefaultGhcArgs 124 | let ghcArgs = baseGhcArgs { gaArgs = gaArgs baseGhcArgs ++ mirrors } 125 | refs <- lift $ do 126 | res <- IO.newIORef [] 127 | let opts = defaultAnalysisOptions { aoMainPkgFallback = "dummyPkg" } 128 | withTypechecked globalLock ghcArgs opts (saveAnalysedTo res) 129 | IO.readIORef res 130 | if null refs 131 | then error "Unexpected: withTypechecked didn't produce any result." 132 | else return $! NonEmpty.fromList refs 133 | where 134 | -- | Needed due to https://ghc.haskell.org/trac/ghc/ticket/14025 to avoid 135 | -- putting output in the input tree (which won't work for a read-only input 136 | -- tree). 137 | makeMirror tmp f = do 138 | -- If 'f' has an absolute path, drop the leading slashes so it can 139 | -- be appended to the tmp root (since "foo" "/bar" == "/bar"). 140 | let newDir = tmp dropWhile (== '/') (File.takeDirectory f) 141 | newFile = newDir File.takeFileName f 142 | Dir.createDirectoryIfMissing True newDir 143 | Dir.copyFile f newFile 144 | return newFile 145 | saveAnalysedTo xs xref = IO.modifyIORef' xs (xref:) 146 | -------------------------------------------------------------------------------- /text-offset/src/Data/Text/Offset.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE BangPatterns, OverloadedStrings, RecordWildCards #-} 16 | -- | Helpers for calculating char/byte offset from line and column number. 17 | -- All line and column offsets in this module are zero-based. 18 | -- 19 | -- Note: currently only line/column to byte offset calculation is exposed, but 20 | -- the 'OffsetTable' structure supports more, which can be added on demand. 21 | 22 | module Data.Text.Offset 23 | ( OffsetTable 24 | -- * Creation 25 | , createOffsetTable 26 | -- * Querying 27 | , lineColToByteOffsetDetail 28 | , lineColToByteOffset 29 | , OffsetError(..) 30 | , OverLineKind(..) 31 | ) where 32 | 33 | import Control.Monad (unless) 34 | import Data.Foldable (foldl') 35 | import Data.Char (ord) 36 | import qualified Data.Text.Lazy as TL 37 | import qualified Data.Vector as V 38 | import qualified Data.Vector.Unboxed as UV 39 | 40 | -- | Unicode code point count (aka number of Haskell Chars). 41 | type CharOffset = Int 42 | -- | Raw byte offset, according to UTF-8 encoding. 43 | type ByteOffset = Int 44 | -- | Offsets at the beginning of a line. 45 | type LineOffsets = (CharOffset, ByteOffset) 46 | 47 | -- | Number of bytes a Char takes when UTF-8 encoded. 48 | type CharBytes = Int 49 | -- | Offset of Char in the line (zero-based) and its encoded byte count (>1). 50 | type MultibyteChar = (Int, CharBytes) 51 | -- | Content char count, excluding the end-of-line separator. 52 | type ContentCharCount = Int 53 | 54 | type LineCharInfo = (ContentCharCount, UV.Vector MultibyteChar) 55 | 56 | -- | Stores info for mapping line/column number (Char-based) to raw offset 57 | -- (either Char or UTF-8 encoded bytes-based). 58 | data OffsetTable = OffsetTable 59 | { lineOffsets :: !(UV.Vector LineOffsets) 60 | , lineCharInfo :: !(V.Vector LineCharInfo) 61 | } deriving (Eq, Show) 62 | 63 | -- | Calculates the 'OffsetTable' for the given text. The table calculation is 64 | -- not optimized (non-strict tuples are lying around), but lookups on the result 65 | -- are performant (since the tuples are stored as unboxed vectors). 66 | createOffsetTable :: TL.Text -> OffsetTable 67 | createOffsetTable 68 | = adjust 69 | . foldl' addLineOffsets [((0, 0), undefined)] 70 | . TL.splitOn "\n" 71 | where 72 | addLineOffsets 73 | :: [(LineOffsets, LineCharInfo)] 74 | -> TL.Text 75 | -> [(LineOffsets, LineCharInfo)] 76 | -- ^ 'LineOffsets' of next line (if any), multibytes of current line. 77 | -- Accumulated in reversed line order. 78 | addLineOffsets ofs@(((cs,bs), _):_) s = 79 | let !charCount = fromIntegral (TL.length s) 80 | multis = multibytes s 81 | !byteCount = bytesUntilCharPosition charCount multis 82 | !newOffs = (cs + charCount + 1, bs + byteCount + 1) 83 | -- +1 for newlines 84 | in (newOffs, (charCount, multis)):ofs 85 | addLineOffsets _ _ = error "addLineOffsets call with empty accumulator" 86 | -- | Drops last offset, and first (bogus) charinfo. 87 | adjust :: [(LineOffsets, LineCharInfo)] -> OffsetTable 88 | adjust xs = 89 | let (ofs, info) = unzip xs 90 | in OffsetTable 91 | { lineOffsets = UV.fromList . reverse . tail $ ofs 92 | , lineCharInfo = V.fromList . tail . reverse $ info 93 | } 94 | 95 | data OffsetError 96 | = NoSuchLine 97 | | EmptyLine 98 | | OverLineEnd !ByteOffset !OverLineKind 99 | -- ^ Clients might want to recover from this case, see 'JustAtLineEnd'. 100 | deriving (Eq, Ord, Show) 101 | 102 | data OverLineKind 103 | = JustAtLineEnd 104 | -- ^ Referenced position is just after the line content. Position can be 105 | -- a valid non-inclusive ending span offset. 106 | | AfterLineEnd 107 | deriving (Eq, Ord, Show) 108 | 109 | -- | Line and col are zero-based, col is in characters. 110 | lineColToByteOffsetDetail 111 | :: OffsetTable -> Int -> Int 112 | -> Either OffsetError Int 113 | lineColToByteOffsetDetail OffsetTable{..} line col = do 114 | (_, lineByteOffs) <- note NoSuchLine (lineOffsets UV.!? line) 115 | (charCount, multis) <- note NoSuchLine (lineCharInfo V.!? line) 116 | unless (charCount > 0) (Left EmptyLine) 117 | let fileOffset = bytesUntilCharPosition col multis + lineByteOffs 118 | unless (col < charCount) $ 119 | let kind = if col == charCount then JustAtLineEnd else AfterLineEnd 120 | in Left $! OverLineEnd fileOffset kind 121 | return $! fileOffset 122 | where 123 | note :: e -> Maybe a -> Either e a 124 | note e = maybe (Left e) Right 125 | 126 | -- | Like 'lineColToByteOffsetDetail', but without failure details. 127 | lineColToByteOffset :: OffsetTable -> Int -> Int -> Maybe Int 128 | lineColToByteOffset t l c = case lineColToByteOffsetDetail t l c of 129 | Left _ -> Nothing 130 | Right a -> Just a 131 | 132 | -- | Byte count up to, but excluding the given char position (zero-based). 133 | bytesUntilCharPosition :: Int -> UV.Vector MultibyteChar -> Int 134 | bytesUntilCharPosition n ms 135 | | UV.null ms = n -- Most of the cases should hit this shortcut. 136 | | otherwise = 137 | let multis = UV.map snd . UV.takeWhile ((< n) . fst) $ ms 138 | !res = n - UV.length multis + UV.sum multis 139 | in res 140 | 141 | -- | Positions and bytecounts of multibyte characters of a text. 142 | multibytes :: TL.Text -> UV.Vector MultibyteChar 143 | multibytes = 144 | UV.fromList . reverse . snd . TL.foldl' addMulti (0, []) 145 | where 146 | addMulti 147 | :: (CharOffset, [MultibyteChar]) -> Char 148 | -> (CharOffset, [MultibyteChar]) 149 | addMulti (coffs, acc) c = 150 | let !n = charBytes c 151 | !acc1 | n == 1 = acc 152 | | otherwise = (coffs, n):acc 153 | !coffs1 = coffs + 1 154 | in (coffs1, acc1) 155 | 156 | -- | Based on 'Data.Text.Internal.Encoding.Utf8'. 157 | charBytes :: Char -> Int 158 | charBytes = ordBytes . ord 159 | where 160 | ordBytes c 161 | | c < 0x80 = 1 162 | | c < 0x0800 = 2 163 | | c < 0x10000 = 3 164 | | otherwise = 4 165 | -------------------------------------------------------------------------------- /kythe-proto/third_party/kythe/kythe/proto/storage.proto: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | 19 | package kythe.proto; 20 | 21 | option java_package = "com.google.devtools.kythe.proto"; 22 | 23 | // VName is a proto representation of a vector name. 24 | // 25 | // Rules: 26 | // - All fields must be optional, and must have default values. 27 | // - No field may ever be removed. If a field is deprecated, it may be 28 | // renamed or marked with a comment, but must not be deleted. 29 | // - New fields are always added to the end of the message. 30 | // - All fields must be strings, not messages. 31 | // 32 | // One of the key principles is that we want as few fields as possible in a 33 | // vname. We're not trying to exhaust the possible dimensions along which a 34 | // name could vary, but to find a minimal basis. Be conservative. 35 | message VName { 36 | // A language-specific signature assigned by the analyzer. 37 | // e.g., "com.google.common.collect.Lists.newLinkedList<#1>()" 38 | string signature = 1; 39 | 40 | // The corpus this name belongs to. 41 | // e.g., "kythe", "chromium", "github.com/creachadair/imath", "aosp" 42 | // The corpus label "kythe" is reserved for internal use. 43 | string corpus = 2; 44 | 45 | // A corpus-specific root label, designating a subordinate collection within 46 | // the corpus. If a corpus stores files in unrelated directory structures, 47 | // for example, the root can be used to distinguish them. Or, of a corpus 48 | // incorporates subprojects, the root can be a project ID that it governs. 49 | // This may also be used to distinguish virtual subgroups of a corpus such as 50 | // generated files. 51 | string root = 3; 52 | 53 | // A path-structured label describing the location of this object relative to 54 | // the corpus and the root. For code, this will generally be the relative 55 | // path to the file containing the code, e.g., "storage/service.go" in kythe. 56 | // 57 | // However, this need not be a true file path; virtual objects like figments 58 | // can assign an ad-hoc abstract ID, or omit it entirely. 59 | // 60 | // Examples: 61 | // "devools/kythe/platform/go/datastore.go" (a file) 62 | // "type/cpp/void.cc" (a type figment) 63 | string path = 4; 64 | 65 | // The language this name belongs to. 66 | // e.g., "c++", "python", "elisp", "haskell", "java" 67 | // 68 | // The schema will define specific labels for each supported language, so we 69 | // don't wind up with a confusion of names like "cxx", "cpp", "C++", etc. 70 | // Prototype: Official language name converted to lowercase. If a version 71 | // number is necessary, include it, e.g., "python3". 72 | string language = 5; 73 | 74 | // Other fields we may need in the future, but do not currently use: 75 | // branch -- a branch name within the corpus depot, e.g., "gslb_branch". 76 | // client -- a source-control client ID, e.g., "sergey:googlex:8:citc". 77 | 78 | // Note: We have intentionally NOT included a revision or timestamp here. 79 | // Time should be recorded as facts belonging to the appropriate Nodes and 80 | // Edges. Having records of when something existed may be important, but time 81 | // is not a good axis for a name -- a name should say "what" something is, not 82 | // "when". So we will store timestamps, revisions, and other markers of this 83 | // kind as facts inside the graph. 84 | } 85 | 86 | message VNameMask { 87 | bool signature = 1; 88 | bool corpus = 2; 89 | bool root = 3; 90 | bool path = 4; 91 | bool language = 5; 92 | } 93 | 94 | // An Entry associates a fact with a graph object (node or edge). This is the 95 | // the primary unit of storage. 96 | message Entry { 97 | VName source = 1; 98 | 99 | // The following two fields must either be both empty, or both nonempty. 100 | string edge_kind = 2; 101 | VName target = 3; 102 | 103 | // The grammar for fact_name: 104 | // name = "/" | 1*path 105 | // path = "/" word 106 | // word = 1*{LETTER|DIGIT|PUNCT} 107 | // LETTER = [A-Za-z] 108 | // DIGIT = [0-9] 109 | // PUNCT = [-.@#$%&_+:()] 110 | string fact_name = 4; 111 | bytes fact_value = 5; 112 | } 113 | 114 | // A collection of Entry instances. 115 | message Entries { 116 | repeated Entry entries = 1; 117 | } 118 | 119 | // Request for a stream of Entry objects from a GraphStore. Read operations 120 | // should be implemented with time complexity proportional to the size of the 121 | // return set. 122 | message ReadRequest { 123 | // Return entries having this source VName, which may not be empty. 124 | VName source = 1; 125 | 126 | // Return entries having this edge kind; if empty, only entries with an empty 127 | // edge kind are returned; if "*", entries of any edge kind are returned. 128 | string edge_kind = 2; 129 | } 130 | 131 | // Request to write Entry objects to a GraphStore 132 | message WriteRequest { 133 | message Update { 134 | string edge_kind = 1; 135 | VName target = 2; 136 | string fact_name = 3; 137 | bytes fact_value = 4; 138 | } 139 | 140 | VName source = 1; 141 | repeated Update update = 2; 142 | } 143 | 144 | // Response to a WriteRequest 145 | message WriteReply {} 146 | 147 | // Request for a stream of Entry objects resulting from a full scan of a 148 | // GraphStore. 149 | message ScanRequest { 150 | // Return entries having this target VName; if empty, any target field is 151 | // matched, including empty. 152 | VName target = 1; 153 | 154 | // Return entries having this kind; if empty, any kind is matched, including 155 | // empty. 156 | string edge_kind = 2; 157 | 158 | // Return entries having fact labels with this prefix; if empty, any fact 159 | // label is matched, 160 | string fact_prefix = 3; 161 | } 162 | 163 | // Request for the size of the shard at the given index. 164 | message CountRequest { 165 | int64 index = 1; 166 | int64 shards = 2; 167 | } 168 | 169 | // Response for a CountRequest 170 | message CountReply { 171 | // Total number of entries in the specified shard. 172 | int64 entries = 1; 173 | } 174 | 175 | // Request for a stream of Entry objects in the given shard. 176 | message ShardRequest { 177 | int64 index = 1; 178 | int64 shards = 2; 179 | } 180 | -------------------------------------------------------------------------------- /haskell-indexer-backend-ghc/tests/Language/Haskell/Indexer/Backend/Ghc/Test/TypeLinkTestBase.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-# LANGUAGE LambdaCase #-} 16 | {-# LANGUAGE OverloadedStrings #-} 17 | module Language.Haskell.Indexer.Backend.Ghc.Test.TypeLinkTestBase (allTests) where 18 | 19 | import Control.Monad.Trans.Reader (ReaderT, runReaderT) 20 | 21 | import Language.Haskell.Indexer.Backend.Ghc.Test.TestHelper 22 | 23 | import Test.Framework (Test) 24 | import Test.Framework.Providers.HUnit (testCase) 25 | import Test.HUnit (assertFailure) 26 | 27 | type AssertionInEnv = ReaderT TestEnv IO () 28 | 29 | testDataVarBinds :: AssertionInEnv 30 | testDataVarBinds = assertXRefsFrom ["typelink/DataVarBinds.hs"] $ 31 | -- Type variable. 32 | declAt (3,9) >>= singleUsage >>= includesPos (3,26) 33 | 34 | testAliasVarBinds :: AssertionInEnv 35 | testAliasVarBinds = assertXRefsFrom ["typelink/AliasVarBinds.hs"] $ 36 | declAt (3,8) >>= singleUsage >>= includesPos (3,13) 37 | 38 | testClassVarBinds :: AssertionInEnv 39 | testClassVarBinds = assertXRefsFrom ["typelink/ClassVarBinds.hs"] $ do 40 | -- Class variable. 41 | declAt (3,9) >>= singleUsage >>= includesPos (4,10) 42 | -- Free variable in class method. 43 | declAt (4,15) >>= usages >>= \case 44 | [u1, u2] -> do 45 | includesPos (4,15) u1 46 | includesPos (4,23) u2 47 | _ -> checking $ assertFailure "Usage count differs" 48 | 49 | testForallBinds :: AssertionInEnv 50 | testForallBinds = assertXRefsFrom ["typelink/ForallBinds.hs"] $ do 51 | -- For now implicit typevars get the declaration emitted at the first 52 | -- reference. Might change in future. 53 | declAt (4,13) >>= usages >>= \case 54 | [u1, u2] -> do 55 | includesPos (4,13) u1 56 | includesPos (4,18) u2 57 | _ -> checking $ assertFailure "Usage count differs" 58 | -- Explicit typevars have the decl nicely on the foralled occurence. 59 | declAt (7,20) >>= usages >>= \case 60 | [u1, u2] -> do 61 | includesPos (7,24) u1 62 | includesPos (7,29) u2 63 | _ -> checking $ assertFailure "Usage count differs" 64 | 65 | testScopedTypeVarBinds :: AssertionInEnv 66 | testScopedTypeVarBinds = assertXRefsFrom ["typelink/ScopedTypeVar.hs"] $ do 67 | declAt (4,13) >>= usages >>= \case 68 | [u1, u2] -> do 69 | includesPos (4,13) u1 70 | includesPos (4,18) u2 71 | _ -> checking $ 72 | assertFailure "Non-scoped top tyvar usage count differs" 73 | declAt (7,12) >>= usages >>= \case 74 | [u1, u2] -> do 75 | includesPos (7,12) u1 76 | includesPos (7,17) u2 77 | _ -> checking $ 78 | assertFailure "Non-scoped local tyvar usage count differs" 79 | declAt (10,18) >>= usages >>= \case 80 | [u1, u2, u3, u4] -> do 81 | includesPos (10,22) u1 82 | includesPos (10,27) u2 83 | includesPos (13,13) u3 84 | includesPos (13,18) u4 85 | _ -> checking $ assertFailure "Scoped usage count differs" 86 | 87 | testInlineSignature :: AssertionInEnv 88 | testInlineSignature = assertXRefsFrom ["typelink/InlineSig.hs"] $ 89 | declAt (4,15) >>= usages >>= \case 90 | [sig, inlineSig] -> do 91 | includesPos (4,19) sig 92 | includesPos (5,20) inlineSig 93 | _ -> checking $ assertFailure "Usage count differs" 94 | 95 | testLetVarBinds :: AssertionInEnv 96 | testLetVarBinds = assertXRefsFrom ["typelink/LetSig.hs"] $ 97 | declAt (4,14) >>= usages >>= \case 98 | [u1, u2] -> do 99 | includesPos (4,14) u1 100 | includesPos (4,19) u2 101 | _ -> checking $ assertFailure "Usage count differs" 102 | 103 | testSimpleSignature :: AssertionInEnv 104 | testSimpleSignature = assertXRefsFrom ["typelink/SimpleSig.hs"] $ do 105 | -- This case is interesting, as the usage is at the top of the 106 | -- type tree (and is missed by using 'universeBi'). 107 | declAt (4,13) >>= singleUsage >>= includesPos (4,13) 108 | -- This case just for completeness. 109 | declAt (7,18) >>= singleUsage >>= includesPos (7,22) 110 | 111 | testDataContextRef :: AssertionInEnv 112 | testDataContextRef = assertXRefsFrom ["typelink/DataContext.hs"] $ 113 | declAt (5,18) >>= usages >>= \case 114 | [u1, u2] -> do 115 | includesPos (5,10) u1 116 | includesPos (5,26) u2 117 | _ -> checking $ assertFailure "Usage count differs" 118 | 119 | testContextBinds :: AssertionInEnv 120 | testContextBinds = assertXRefsFrom ["typelink/ContextBinds.hs"] $ do 121 | -- In the unscoped case, we put the decl on the first occurence, 122 | -- including the context. It's somewhat arbitrary, but at least 123 | -- makes sure we don't miss the context. 124 | declAt (4,11) >>= usages >>= \case 125 | [u1, u2] -> do 126 | includesPos (4,11) u1 127 | includesPos (4,16) u2 128 | _ -> checking $ assertFailure "Unscoped usage count differs" 129 | -- This is a fun case where a type var only appears in the context. 130 | declAt (7,12) >>= usages >>= \case 131 | [u1, u2] -> do 132 | includesPos (7,12) u1 133 | includesPos (7,15) u2 134 | _ -> checking $ assertFailure "Context-only usage count differs" 135 | declAt (10,15) >>= usages >>= \case 136 | [u1, u2] -> do 137 | includesPos (10,23) u1 138 | includesPos (10,29) u2 139 | _ -> checking $ assertFailure "Scoped usage count differs" 140 | 141 | 142 | -- | Prepares the tests to run with the given test environment. 143 | allTests :: TestEnv -> [Test] 144 | allTests env = 145 | [ envTestCase "data-var-binds" testDataVarBinds 146 | , envTestCase "alias-var-binds" testAliasVarBinds 147 | , envTestCase "class-var-binds" testClassVarBinds 148 | , envTestCase "forall-binds" testForallBinds 149 | , envTestCase "scoped-var-binds" testScopedTypeVarBinds 150 | , envTestCase "inline-signature" testInlineSignature 151 | , envTestCase "let-var-binds" testLetVarBinds 152 | , envTestCase "simple-signature" testSimpleSignature 153 | , envTestCase "data-context-ref" testDataContextRef 154 | , envTestCase "context-binds" testContextBinds 155 | ] 156 | where 157 | envTestCase name test = testCase name (runReaderT test env) 158 | -------------------------------------------------------------------------------- /kythe-schema/src/Language/Kythe/Schema/Typed.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-| Convenience module for building Kythe facts and edges. 16 | Provides slightly more safety over the raw structures in 'Kythe.Schema.Raw' 17 | by using enumerations and types. 18 | 19 | The set of available nodes/edges is not exhaustive, and only concerned about 20 | encoding the ones needed for indexing Haskell. Once the Kythe schema is 21 | finalized, the module can be refactored to full support. 22 | 23 | Note: during development, more aggressive type restrictions were considered, 24 | but the power-to-code ratio was inadequate. Mostly due to the Kythe analysis 25 | happening in a distributed way, so full info about a target node is not 26 | available. 27 | -} 28 | {-# LANGUAGE GADTs #-} 29 | {-# LANGUAGE LambdaCase #-} 30 | {-# LANGUAGE OverloadedStrings #-} 31 | module Language.Kythe.Schema.Typed 32 | ( fact 33 | , Fact(..) 34 | , AnonymousFact 35 | -- Nodes 36 | , NodeKind(..) 37 | , nodeFacts, nodeFact 38 | -- Facts 39 | , AnchorSubkind(..) 40 | , HowComplete(..) 41 | -- Edges 42 | , edge 43 | , Edge(..), AnchorEdge(..) 44 | ) where 45 | 46 | import Data.Text (Text, pack) 47 | import Data.Text.Encoding (encodeUtf8) 48 | import Data.ByteString (ByteString) 49 | 50 | import qualified Language.Kythe.Schema.Raw as Raw 51 | 52 | -- | Node kinds allowed by schema. 53 | data NodeKind 54 | = AnchorNK 55 | | FileNK 56 | | PackageNK 57 | | VariableNK 58 | 59 | -- | Renders node kind to Kythe text value. 60 | printNodeKind :: NodeKind -> Text 61 | printNodeKind = \case 62 | AnchorNK -> "anchor" 63 | FileNK -> "file" 64 | PackageNK -> "package" 65 | VariableNK -> "variable" 66 | 67 | -- | Name of a node fact constrained by the accepted value type. 68 | -- 69 | -- There can be multiple, node kind dependent fact names referring to the same 70 | -- raw Kythe fact name, if the value type can differ depending on the node kind. 71 | -- This is most notably true for the "text" fact. 72 | data Fact value where 73 | AnchorSubkindF :: Fact AnchorSubkind 74 | NodeKindF :: Fact NodeKind 75 | FileTextF :: Fact ByteString 76 | LocStartF :: Fact Int 77 | LocEndF :: Fact Int 78 | CompleteF :: Fact HowComplete 79 | SnippetStartF :: Fact Int 80 | SnippetEndF :: Fact Int 81 | DocUriF :: Fact ByteString 82 | 83 | -- | Renders a fact name to Kythe text value. 84 | printFact :: Fact v -> Text 85 | printFact = \case 86 | AnchorSubkindF -> "/kythe/subkind" 87 | NodeKindF -> "/kythe/node/kind" 88 | FileTextF -> "/kythe/text" 89 | LocStartF -> "/kythe/loc/start" 90 | LocEndF -> "/kythe/loc/end" 91 | CompleteF -> "/kythe/complete" 92 | SnippetStartF -> "/kythe/snippet/start" 93 | SnippetEndF -> "/kythe/snippet/end" 94 | DocUriF -> "/kythe/doc/uri" 95 | 96 | -- | How complete is the declaration / definition. 97 | data HowComplete 98 | = Incomplete -- ^ Incomplete, can't be used yet. 99 | | Complete -- ^ Can be used, but not fully defined. 100 | | Definition -- ^ Fully defined. 101 | 102 | -- | Anchors can be implicit. 103 | data AnchorSubkind = ImplicitAnchor 104 | 105 | -- | Constructs an emittable raw fact. 106 | -- 107 | -- The types don't protect against using the wrong fact for a given node kind. 108 | -- That was considered but too noisy to be practical on small scale usage. 109 | fact :: (FactValuePrintable v) => Raw.VName -> Fact v -> v -> Raw.Fact 110 | fact vname f v = Raw.Fact vname (printFact f) (printFactValue v) 111 | 112 | -- | Convenience for describing all facts about a node. Saves passing the vname 113 | -- arg each time, compared to 'fact'. Also automatically adds the node kind 114 | -- fact. 115 | -- 116 | -- Example usage: 117 | -- 118 | -- nodeFacts vname FileNK [ nodeFact FileTextF contentBytes, ... ] 119 | -- 120 | nodeFacts :: Raw.VName -> NodeKind -> [AnonymousFact] -> [Raw.Entry] 121 | nodeFacts vname nk = map (Raw.FactE . go) . (nkFact:) 122 | where 123 | go (AnyFact rf rv) = Raw.Fact vname rf rv 124 | nkFact = nodeFact NodeKindF nk 125 | 126 | -- | To be used with 'nodeFacts'. 127 | nodeFact :: (FactValuePrintable v) => Fact v -> v -> AnonymousFact 128 | nodeFact f v = AnyFact (printFact f) (printFactValue v) 129 | 130 | -- | Glorified pair. 131 | data AnonymousFact = AnyFact !Raw.FactName !Raw.FactValue 132 | 133 | -- | Describes how to render fact values to raw Kythe bytes. 134 | class FactValuePrintable a where 135 | printFactValue :: a -> Raw.FactValue 136 | 137 | instance FactValuePrintable ByteString where 138 | printFactValue = id 139 | 140 | instance FactValuePrintable Text where 141 | printFactValue = encodeUtf8 142 | 143 | instance FactValuePrintable NodeKind where 144 | printFactValue = encodeUtf8 . printNodeKind 145 | 146 | instance FactValuePrintable Int where 147 | -- TODO(robinpalotai): BS Builder 148 | printFactValue = encodeUtf8 . pack . show 149 | 150 | instance FactValuePrintable HowComplete where 151 | printFactValue = \case 152 | Incomplete -> "incomplete" 153 | Complete -> "complete" 154 | Definition -> "definition" 155 | 156 | instance FactValuePrintable AnchorSubkind where 157 | printFactValue = \case 158 | ImplicitAnchor -> "implicit" 159 | 160 | -- | Emits an edge entry. To be used directly. 161 | edge :: Raw.VName -> Edge -> Raw.VName -> Raw.Entry 162 | edge source e target = Raw.EdgeE (Raw.Edge source (printEdge e) target) 163 | 164 | data Edge 165 | = ChildOfE 166 | | OverridesE 167 | | OverridesRootE 168 | | ExtendsE -- ^ Note: Kythe allows emitting "extends/"-prefixed custom 169 | -- edges. No need for that, yet. 170 | | AnchorEdgeE AnchorEdge 171 | 172 | -- | Edges running from anchor to semantic node. 173 | data AnchorEdge 174 | = DefinesE 175 | | DefinesBindingE 176 | | RefE 177 | | RefCallE 178 | | RefDocE 179 | | RefImportsE 180 | 181 | printEdge :: Edge -> Text 182 | printEdge = \case 183 | ChildOfE -> "/kythe/edge/childof" 184 | OverridesE -> "/kythe/edge/overrides" 185 | OverridesRootE -> "/kythe/edge/overrides/root" 186 | ExtendsE -> "/kythe/edge/extends" 187 | AnchorEdgeE e -> case e of 188 | DefinesE -> "/kythe/edge/defines" 189 | DefinesBindingE -> "/kythe/edge/defines/binding" 190 | RefE -> "/kythe/edge/ref" 191 | RefCallE -> "/kythe/edge/ref/call" 192 | RefDocE -> "/kythe/edge/ref/doc" 193 | RefImportsE -> "/kythe/edge/ref/imports" 194 | 195 | -------------------------------------------------------------------------------- /haskell-indexer-pipeline-ghckythe-wrapper/src/Language/Haskell/Indexer/Args.hs: -------------------------------------------------------------------------------- 1 | 2 | -- Copyright 2017 Google Inc. 3 | -- 4 | -- Licensed under the Apache License, Version 2.0 (the "License"); 5 | -- you may not use this file except in compliance with the License. 6 | -- You may obtain a copy of the License at 7 | -- 8 | -- http://www.apache.org/licenses/LICENSE-2.0 9 | -- 10 | -- Unless required by applicable law or agreed to in writing, software 11 | -- distributed under the License is distributed on an "AS IS" BASIS, 12 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | -- See the License for the specific language governing permissions and 14 | -- limitations under the License. 15 | 16 | {-# LANGUAGE CPP #-} 17 | {-# LANGUAGE OverloadedStrings #-} 18 | {-# LANGUAGE RecordWildCards #-} 19 | {- 20 | This module provides the wrapper executable function (wrapperMain) but also 21 | exposes the flag parsing logic so it can be reused by the plugin. 22 | -} 23 | module Language.Haskell.Indexer.Args where 24 | 25 | import Data.Bits ((.|.), (.&.), shiftR) 26 | import Data.Bool (bool) 27 | import qualified Data.ByteString as B 28 | import qualified Data.ByteString.Lazy as BL 29 | import qualified Data.ByteString.Builder as Builder 30 | import Data.Maybe (fromMaybe) 31 | #if !MIN_VERSION_base(4,11,0) 32 | import Data.Monoid ((<>)) 33 | #endif 34 | import Data.ProtoLens (encodeMessage) 35 | import Data.Text (Text) 36 | import qualified Data.Text as T 37 | 38 | import Control.Concurrent.MVar (newMVar) 39 | import Control.Exception (throwIO, ErrorCall(..)) 40 | import Options.Applicative 41 | import System.Environment (getArgs, withArgs) 42 | 43 | import Language.Haskell.Indexer.Backend.AnalysisOptions (AnalysisOptions(..)) 44 | import Language.Haskell.Indexer.Backend.GhcArgs 45 | import qualified Language.Kythe.Schema.Raw as Raw 46 | import qualified Language.Kythe.Schema.Raw.Proto as Raw 47 | import Language.Haskell.Indexer.Pipeline.GhcKythe (ghcToKythe, pluginContinuation) 48 | import Language.Haskell.Indexer.Util.Path (asTextPath, stripTmpPrefix) 49 | import Language.Haskell.Indexer.Translate 50 | 51 | import DynFlags (defaultFatalMessager, defaultFlushOut) 52 | import GHC (defaultErrorHandler) 53 | import GHC.IO.Handle 54 | 55 | -- | Command-line flags to control the indexer behavior. 56 | data Flags = Flags 57 | { flagCorpus :: !Text 58 | , flagMainPackageRename :: !(Maybe Text) 59 | , flagStripPathPrefix :: !(Maybe Text) 60 | , flagPrependPathPrefix :: !(Maybe Text) 61 | , flagKeepTempPathPrefix :: !Bool 62 | , flagOverridePgmP :: !(Maybe FilePath) 63 | , flagOverrideLibdir :: !(Maybe FilePath) 64 | -- Path to a directory where to place the output files when running 65 | -- in plugin mode. 66 | , flagOutput :: !(Maybe FilePath) 67 | } 68 | 69 | wrapperMain :: IO () 70 | wrapperMain = do 71 | (wrapperArgs, rest) <- break (== "--") <$> getArgs 72 | case rest of 73 | _:ghcArgs -> withArgs wrapperArgs (execParser opts) >>= index ghcArgs 74 | _ -> throwIO (ErrorCall "No -- found in args.") 75 | where 76 | opts = info (helper <*> flagParser) 77 | $ header ( 78 | "ghc_wrapper - pretends to be GHC and writes Kythe artifacts. " 79 | ++ "Options after the first -- are passed on to GHC.") 80 | <> fullDesc 81 | 82 | kythePlugin :: Handle -> (AnalysisOptions -> IO XRef) -> Flags -> IO () 83 | kythePlugin h f flags = 84 | indexX (pluginContinuation f h) [] flags 85 | 86 | wrapperParser :: [String] -> IO Flags 87 | wrapperParser opts = withArgs opts (execParser (info flagParser fullDesc)) 88 | 89 | flagParser :: Parser Flags 90 | flagParser = Flags 91 | <$> (T.pack <$> strOption 92 | ( long "corpus" 93 | <> short 'c' 94 | <> help "The name of the Kythe corpus being indexed." 95 | <> value "" 96 | <> showDefault)) 97 | <*> optional (T.pack <$> strOption 98 | ( long "rename_main" 99 | <> short 'm' 100 | <> metavar "PACKAGE" 101 | <> help ("Changes the 'main' package name when emitting entries. " 102 | ++ "Useful when indexing multiple binaries."))) 103 | <*> optional (T.pack <$> strOption 104 | ( long "drop_path_prefix" 105 | <> metavar "PREFIX" 106 | <> help "Strip the given prefix from emitted filepaths.")) 107 | <*> optional (T.pack <$> strOption 108 | ( long "prepend_path_prefix" 109 | <> metavar "PREFIX" 110 | <> help "Prepends prefix to emitted filepaths (after stripping, if any).")) 111 | <*> switch 112 | ( long "keep_path_tmp_prefix" 113 | <> help ("If set, won't apply the default removal of temporary " 114 | ++ "dirs from emitted path prefixes.")) 115 | <*> optional (strOption 116 | ( long "pgmP_binary" 117 | <> short 'P' 118 | <> metavar "PATH" 119 | <> help ("Overrides the preprocessor binary, but keeps it's " 120 | ++ "options. Note: other tools can still be overriden " 121 | ++ "by passing the regular -pgmX GHC options."))) 122 | <*> optional (strOption 123 | ( long "libdir" 124 | <> short 'B' 125 | <> metavar "PATH" 126 | <> help ("Overrides the GHC libdir."))) 127 | <*> optional (strOption 128 | ( long "output" 129 | <> short 'o' 130 | <> metavar "INDEX_OUT_DIR" 131 | <> help ("The directory to write the indices to in plugin mode. " 132 | ++ "Normal mode emits entry stream to stdout."))) 133 | 134 | index :: [String] -> Flags -> IO () 135 | index args fs = do 136 | lock <- newMVar () 137 | withErrorHandler $ indexX (ghcToKythe lock) args fs 138 | where 139 | withErrorHandler :: IO a -> IO a 140 | withErrorHandler = defaultErrorHandler defaultFatalMessager defaultFlushOut 141 | 142 | indexX 143 | :: (GhcArgs -> AnalysisOptions -> Raw.VName 144 | -> (Handle -> [Raw.Entry] -> IO ()) -> IO ()) 145 | -> [String] -> Flags -> IO () 146 | indexX k args Flags{..} = do 147 | let ghcArgs = GhcArgs 148 | { gaArgs = args 149 | , gaToolOverride = ToolOverride 150 | { overridePgmP = flagOverridePgmP 151 | } 152 | , gaLibdirOverride = OverrideLibdir <$> flagOverrideLibdir 153 | } 154 | analysisOptions = AnalysisOptions 155 | { aoMainPkgFallback = fromMaybe "main" flagMainPackageRename 156 | , aoDemanglePackageName = id 157 | , aoFilePathTransform 158 | = customPrepend 159 | . customStrip 160 | . bool (asTextPath stripTmpPrefix) id flagKeepTempPathPrefix 161 | } 162 | where customStrip p = fromMaybe p $ do 163 | prefix <- flagStripPathPrefix 164 | T.stripPrefix prefix p 165 | customPrepend p = fromMaybe p . fmap (<> p) 166 | $ flagPrependPathPrefix 167 | baseVName = Raw.VName "" flagCorpus "" "" "haskell" 168 | k ghcArgs analysisOptions baseVName collect 169 | where 170 | collect handle = mapM_ (\m -> do 171 | let wire = encodeMessage . Raw.toEntryProto $ m 172 | B.hPutStr handle . BL.toStrict . Builder.toLazyByteString 173 | . varInt . B.length $ wire 174 | B.hPutStr handle wire) 175 | 176 | -- | From proto-lens. 177 | varInt :: Int -> Builder.Builder 178 | varInt n 179 | | n < 128 = Builder.word8 (fromIntegral n) 180 | | otherwise = Builder.word8 (fromIntegral $ n .&. 127 .|. 128) 181 | <> varInt (n `shiftR` 7) 182 | -------------------------------------------------------------------------------- /haskell-indexer-translate/src/Language/Haskell/Indexer/Translate.hs: -------------------------------------------------------------------------------- 1 | -- Copyright 2017 Google Inc. 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | {-| Intermediate data structure from GhcAnalyser. 16 | This will be transformed to actual Grok/Kythe schema. 17 | 18 | We need to keep this relatively lightweight to avoid mirroring GHC AST 19 | structures. The goal for now should be support for code browsing and 20 | crossreferencing, _not_ faithfully mirroring exact types for static analysis. 21 | 22 | Should not expose info about the producer (GHC), to make them switchable. 23 | 24 | Should not make assumptions about the consuming backend for same reason. 25 | -} 26 | module Language.Haskell.Indexer.Translate 27 | ( SourcePath(..) 28 | , Pos(..), Span(..), spanFile 29 | , Tick(..) 30 | -- 31 | , XRef(..) 32 | , AnalysedFile(..) 33 | , ModuleTick(..) 34 | , Decl(..) 35 | , DeclExtra(..), emptyExtra, withExtra 36 | , DocUriDecl(..) 37 | , ModuleDocUriDecl(..) 38 | , PkgModule(..) 39 | , StringyType(..) 40 | , TickReference(..), ReferenceKind(..) 41 | , Relation(..), RelationKind(..) 42 | ) where 43 | 44 | import Data.Maybe (fromMaybe) 45 | import Data.Text (Text) 46 | 47 | -- | Like 'FilePath', but uses 'Text' and is a real wrapper, not a type alias. 48 | -- Note: source paths usually live in a temporary workdir, or maybe under some 49 | -- top-level dir. If possible, feed this top-level dir to the frontend, and 50 | -- it should strip it from the emitted paths. 51 | newtype SourcePath = SourcePath { unSourcePath :: Text } 52 | deriving (Eq, Ord) 53 | 54 | instance Show SourcePath where 55 | show (SourcePath p) = show p 56 | 57 | -- | Line, col and filename. Col is in characters (not bytes). 58 | data Pos = Pos !Int !Int !SourcePath 59 | deriving (Eq, Ord, Show) 60 | 61 | -- | A text range with start and end position (inclusive/exclusive 62 | -- respectively). 63 | data Span = Span !Pos !Pos 64 | deriving (Eq, Ord, Show) 65 | 66 | -- | File containing Span. 67 | spanFile :: Span -> SourcePath 68 | spanFile (Span (Pos _ _ f) _) = f 69 | 70 | -- | All data for a given input file. TODO rename? 71 | -- Contains lists, to give lazy evaluation a chance and results can eventually 72 | -- be streamed with lower peek memory residency. 73 | data XRef = XRef 74 | { xrefFile :: !AnalysedFile 75 | , xrefModule :: !ModuleTick 76 | , xrefDecls :: [Decl] 77 | , xrefDocDecls :: [DocUriDecl] 78 | , xrefModuleDocDecls :: [ModuleDocUriDecl] 79 | , xrefCrossRefs :: [TickReference] 80 | , xrefRelations :: [Relation] 81 | , xrefImports :: [ModuleTick] 82 | } 83 | deriving (Eq, Show) 84 | 85 | data AnalysedFile = AnalysedFile 86 | { analysedTempPath :: !SourcePath 87 | -- ^ The path of the analysed file on the actual filesystem, not 88 | -- yet stripped of the temporary directory prefix. 89 | -- The frontend can use this path to read the file contents if needed. 90 | , analysedOriginalPath :: !SourcePath 91 | -- ^ A nice, abstract path, which developers think of as the location of 92 | -- the file. Ideally stripped of temporary workdirs. 93 | } 94 | deriving (Eq, Show) 95 | 96 | -- | Info required to reference a module. 97 | data ModuleTick = ModuleTick 98 | { mtPkgModule :: !PkgModule 99 | , mtSpan :: !(Maybe Span) 100 | -- ^ Span of the module name. 101 | -- For example, 'X' in 'module X where'. 102 | -- Main modules can have this missing. 103 | } 104 | deriving (Eq, Ord, Show) 105 | 106 | -- | A Tick(et) is a globally unique identifier for some entity in the AST. 107 | -- Not mandatory, but is ideally stable across multiple compilations. 108 | -- 109 | -- Not related to GHC's SCC annotations (called ticks internally). 110 | data Tick = Tick 111 | { tickSourcePath :: !(Maybe SourcePath) 112 | , tickPkgModule :: !PkgModule 113 | , tickThing :: !Text 114 | -- ^ The unqualified name of the entity. 115 | , tickSpan :: !(Maybe Span) 116 | -- ^ Can be broader or just loosely connected to the physical location 117 | -- of the entity in source. Should only be used for generating a unique 118 | -- tick string. Use other spans in Decl for source linking. 119 | , tickUniqueInModule :: !Bool 120 | -- ^ If true, the generated unique name can omit the span. 121 | -- This usually signals top levelness too. 122 | -- TODO(robinpalotai): make the distinction clear? Rename? 123 | , tickTermLevel :: !Bool 124 | -- ^ Needed to disambiguate same name occuring in term and type level. 125 | } 126 | deriving (Eq, Ord, Show) 127 | 128 | data PkgModule = PkgModule 129 | { getPackage :: !Text 130 | , getModule :: !Text 131 | , getPackageWithVersion :: !Text 132 | } 133 | deriving (Eq, Ord, Show) 134 | 135 | data Decl = Decl 136 | { declTick :: !Tick 137 | , declIdentifierSpan :: !(Maybe Span) 138 | -- ^ Points to a potentially narrow span containing the identifier of 139 | -- the decl. Also see other spanny fields in DeclExtra. 140 | , declType :: !StringyType 141 | -- ^ Should go away once we switch to emitting separate type Decls. 142 | , declExtra :: !(Maybe DeclExtra) 143 | -- ^ Since rarely present, in 'Maybe' to minimize memory usage, and to 144 | -- let DeclExtra grow without concern. 145 | } 146 | deriving (Eq, Show) 147 | 148 | data DocUriDecl = DocUriDecl 149 | { ddeclTick :: !Tick 150 | , ddeclDocUri :: !Text -- ^ Document URI for the ticket. 151 | } 152 | deriving (Eq, Show) 153 | 154 | data ModuleDocUriDecl = ModuleDocUriDecl 155 | { mddeclTick :: !ModuleTick 156 | , mddeclDocUri :: !Text -- ^ Document URI for the module. 157 | } 158 | deriving (Eq, Show) 159 | 160 | -- | Additional information about the decl, for info that is rarely present. 161 | data DeclExtra = DeclExtra 162 | { methodForInstance :: !(Maybe Text) 163 | -- ^ A readable unqualified name of the instance, in the form of 164 | -- "Cls Inst". Frontends can use this data to provide more descripive 165 | -- identifier name ("Cls Inst.foo" instead just "foo"), which is 166 | -- helpful when listed in an UI. 167 | , alternateIdSpan :: !(Maybe Span) 168 | -- ^ Set if the declIdentifierSpan overlaps with other spans, making it 169 | -- problematic for UI tools. Then the alternateIdSpan can be used by 170 | -- frontends for example for hyperlinking. 171 | } 172 | deriving (Eq, Show) 173 | 174 | emptyExtra :: DeclExtra 175 | emptyExtra = DeclExtra Nothing Nothing 176 | 177 | -- | Modifies declExtra of the given decl, and creates one if it is Nothing so 178 | -- far. 179 | withExtra :: (DeclExtra -> DeclExtra) -> Decl -> Decl 180 | withExtra f d = 181 | let extra = fromMaybe emptyExtra (declExtra d) 182 | in d { declExtra = Just $! f extra } 183 | 184 | -- | Ideally types of things are referred using the tick of those types. 185 | -- But for complex types, such a tick (and the accompanying decl) must be 186 | -- assembled (recursively) on the spot. This is not yet done, in that case 187 | -- we lie that this is just a simple type (represented by the given string). 188 | -- The loss with that is that the type graph is unusable for programmatic 189 | -- queries. 190 | -- TODO(robinpalotai): remove this and add a proper Decl for the type + 191 | -- a RelationKind ctor 'IsTypeOf'. 192 | data StringyType = StringyType 193 | { declQualifiedType :: !Text 194 | , declUserFriendlyType :: !Text 195 | } 196 | deriving (Eq, Show) 197 | 198 | -- | Reference to the given tick from the given span. 199 | data TickReference = TickReference 200 | { refTargetTick :: !Tick 201 | , refSourceSpan :: !Span 202 | -- ^ The precise location of the reference. Frontends probably want to 203 | -- make this a hyperlink on the UI. 204 | , refHighLevelContext :: !(Maybe Tick) 205 | -- ^ The context from which the reference originates. Theoretically a 206 | -- frontend could infer the context (enclosing scope) from the reference 207 | -- source span, but 1) it is not obvious how large context to choose, 208 | -- and 2) since the compiler already has the scoping info, it is easier 209 | -- for the indexer to emit it. 210 | -- 211 | -- Here we pragmatically set the context to the current top-level 212 | -- function, if any. On the UI, this might show up as the next element 213 | -- in the call chain - see 'ReferenceKind'. 214 | , refKind :: !ReferenceKind 215 | } deriving (Eq, Show) 216 | 217 | -- | Distinguishing a plain reference from a call is traditional in imperative 218 | -- languages, but in a functional language - or an imperative one with 219 | -- functional elements - these concepts are fuzzy. For example, we might see 220 | -- partial application as either reference or call. 221 | 222 | -- Disclaimer: author of this comment is not an expert on the topic, so 223 | -- following might be imprecise or even foolish. 224 | -- 225 | -- Traditionally a call puts a new entry on the call stack. 226 | -- But with partial application or lazy evaluation our runtime representation 227 | -- is a call graph (with thunks as nodes) rather than a linear stack. 228 | -- 229 | -- So here we instead resort to considering what a call means for the user. 230 | -- An UI with code crossref support can typically display the call stack or 231 | -- possible call chain of a function. This chain is expanded by the user to 232 | -- discover sites that deal with that function - an example situation can be 233 | -- debugging a problem or understanding new code. 234 | -- 235 | -- Here we adopt a simplistic distinction - call is anything with at least one 236 | -- argument application, the rest are reference. The frontend is free to 237 | -- disregard this information and treat everything as calls or references 238 | -- though. 239 | data ReferenceKind 240 | = Ref -- ^ Reference 241 | | Call -- ^ Function call 242 | | TypeDecl -- ^ Usage of identifier in type declaration, left to "::" 243 | | Import -- ^ Imported entities 244 | deriving (Eq, Ord, Show) 245 | 246 | -- | A Relation is between standalone semantic nodes, in contrast to 247 | -- TickReference, which is between a source span and a semantic node. 248 | -- 249 | -- Read it aloud as 'relSourceTick' 'relKind' 'relTargetTick'. 250 | data Relation = Relation 251 | { relSourceTick :: !Tick 252 | , relKind :: !RelationKind 253 | , relTargetTick :: !Tick 254 | } 255 | deriving (Eq, Show) 256 | 257 | data RelationKind = ImplementsMethod | InstantiatesClass 258 | deriving (Eq, Ord, Show) 259 | -------------------------------------------------------------------------------- /kythe-proto/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 The haskell-indexer Authors. All rights reserved. 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS 179 | 180 | APPENDIX: How to apply the Apache License to your work. 181 | 182 | To apply the Apache License to your work, attach the following 183 | boilerplate notice, with the fields enclosed by brackets "[]" 184 | replaced with your own identifying information. (Don't include 185 | the brackets!) The text should be enclosed in the appropriate 186 | comment syntax for the file format. We also recommend that a 187 | file or class name and description of purpose be included on the 188 | same "printed page" as the copyright notice for easier 189 | identification within third-party archives. 190 | 191 | Copyright 2017, The haskell-indexer Authors. 192 | 193 | Licensed under the Apache License, Version 2.0 (the "License"); 194 | you may not use this file except in compliance with the License. 195 | You may obtain a copy of the License at 196 | 197 | http://www.apache.org/licenses/LICENSE-2.0 198 | 199 | Unless required by applicable law or agreed to in writing, software 200 | distributed under the License is distributed on an "AS IS" BASIS, 201 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 202 | See the License for the specific language governing permissions and 203 | limitations under the License. 204 | --------------------------------------------------------------------------------