= getViterbiParseFromChart(finalState, chart);
173 | const toProbability = grammar.probabilityMapping.toProbability;
174 | const finalScore = chart.getViterbiScore(finalState).innerScore;
175 |
176 | return {
177 | parseTree,
178 | probability: toProbability(finalScore)
179 | };
180 | }
--------------------------------------------------------------------------------
/src/earley/parsetree.ts:
--------------------------------------------------------------------------------
1 | import { Category } from "../grammar/category";
2 |
3 | /**
4 | * A parse tree that represents the derivation of a string based on the
5 | * rules in a {@link Grammar}. Parse trees recursively contain
6 | * other parse trees, so they can be iterated through to
7 | * find the entire derivation of a category.
8 | *
9 | * Parse trees are essentially partial views of a Chart from a
10 | * given {@link State} or {@link Category}. They represent the completed
11 | * category at a given string index and origin position.
12 | */
13 | export interface ParseTree {
14 | category: Category;
15 | children: ParseTree[];
16 | token?: T;
17 | }
18 |
19 | //noinspection JSUnusedGlobalSymbols
20 | /**
21 | * Creates a new parse tree with the specified category, parent, and
22 | * child trees.
23 | *
24 | * @param category The category of the category of this parse
25 | * tree.
26 | * @param children The list of children of this parse tree, in their linear
27 | * order.
28 | */
29 | export function createParseTree(category: Category, children: ParseTree[] = []): ParseTree {
30 | return {category, children};
31 | }
32 |
33 | export function addRightMost(addTo: ParseTree, addMe: ParseTree) {
34 | addTo.children.push(addMe);
35 | }
36 |
37 |
38 | // public static class Token extends ParseTree {
39 | // public final org.leibnizcenter.cfg.token.Token token;
40 | //
41 | // public Token(org.leibnizcenter.cfg.token.Token scannedToken, Category category) {
42 | // super(category, null);
43 | // this.token = scannedToken;
44 | // }
45 | //
46 | // public Token(ScannedTokenState scannedState) {
47 | // this(scannedState.scannedToken, scannedState.scannedCategory);
48 | // }
49 | //
50 | //
51 | // @Override
52 | // public int hashCode() {
53 | // return super.hashCode() + token.hashCode();
54 | // }
55 | //
56 | // @Override
57 | // public boolean equals(Object o) {
58 | // return o instanceof Token && super.equals(o) && token.equals(((Token) o).token);
59 | // }
60 | // }
61 | //
62 | // public static class NonToken extends ParseTree {
63 | // public NonToken(Category node) {
64 | // super(node);
65 | // }
66 | //
67 | // public NonToken(Category node, LinkedList children) {
68 | // super(node, children);
69 | // }
70 | //
71 | // @Override
72 | // public boolean equals(Object o) {
73 | // return o instanceof NonToken && super.equals(o);
74 | // }
75 | // }
76 | // }
--------------------------------------------------------------------------------
/src/earley/predict.ts:
--------------------------------------------------------------------------------
1 | import {Grammar} from "../grammar/grammar";
2 | import {Chart} from "./chart/chart";
3 | import {State, getActiveCategory} from "./chart/state";
4 | import {Category, isNonTerminal, NonTerminal} from "../grammar/category";
5 | import {Rule} from "../grammar/rule";
6 |
7 | /**
8 | * Makes predictions in the specified chart at the given index.
9 |
10 | * For each chart at position i, look at the the nonterminal at the dot position,
11 | * plus a chart that expands that nonterminal at position i, with the dot position at 0
12 | *
13 | * @param index The string index to make predictions at.
14 | * @param grammar
15 | * @param stateSets
16 | */
17 | export function predict(index: number,
18 | grammar: Grammar,
19 | stateSets: Chart) {
20 | const changes: any[] = [];
21 | const statesToPredictOn: Set> = stateSets.getStatesActiveOnNonTerminals(index);
22 | if (statesToPredictOn) {
23 | const newStates = new Set>();
24 | const probMap = grammar.probabilityMapping;
25 | const sr = probMap.semiring;
26 | const fromProb = probMap.fromProbability;
27 | // O(|stateset(i)|) = O(|grammar|): For all states i: Xk → λ·Zμ...
28 | statesToPredictOn.forEach((statePredecessor: State) => {
29 | const Z: Category = getActiveCategory(statePredecessor);
30 | const prevForward: S = stateSets.getForwardScore(statePredecessor);
31 |
32 | // For all productions Y → v such that R(Z =*L> Y) is nonzero
33 | grammar.leftStarCorners
34 | .getNonZeroScores(Z)
35 | .forEach((Y: Category) => {
36 | // TODO ? can be more efficient by indexing on Y?
37 | if (isNonTerminal(Y) && grammar.ruleMap.has(Y))
38 | grammar.ruleMap.get(Y).forEach((Y_to_v: Rule) => {
39 | // we predict chart i: Yi → ·v
40 | // noinspection JSSuspiciousNameCombination
41 | const Y: NonTerminal = Y_to_v.left;
42 |
43 |
44 | // γ' = P(Y → v)
45 | const Y_to_vScore: S = fromProb(Y_to_v.probability);
46 |
47 | // α' = α * R(Z =*L> Y) * P(Y → v)
48 | const fw: S = sr.times(
49 | prevForward,
50 | sr.times(
51 | fromProb(grammar.getLeftStarScore(Z, Y)),
52 | Y_to_vScore
53 | )
54 | );
55 |
56 | let predicted: State;
57 |
58 | // We might want to increment the probability of an existing chart
59 | const isNew = !stateSets.has(Y_to_v, index, index, 0);
60 | predicted = isNew ? {
61 | position: index,
62 | ruleStartPosition: index,
63 | ruleDotPosition: 0,
64 | rule: Y_to_v
65 | } : stateSets.getOrCreate(index, index, 0, Y_to_v);
66 | if (isNew) // save for later
67 | newStates.add(predicted);
68 |
69 | const innerScore: S = stateSets.getInnerScore(predicted);
70 | if (!(Y_to_vScore === innerScore || probMap.ZERO === innerScore))throw new Error(Y_to_vScore + " != " + innerScore);
71 |
72 | const viterbi = {
73 | origin: statePredecessor,
74 | resultingState: predicted,
75 | innerScore: Y_to_vScore,
76 | };
77 |
78 | stateSets.addForwardScore(predicted, fw, sr);
79 | stateSets.setInnerScore(predicted, Y_to_vScore);
80 | stateSets.setViterbiScore(viterbi);
81 |
82 |
83 | const change = {
84 | state: predicted,
85 | innerScore: Y_to_vScore,
86 | forwardScore: fw,
87 | viterbiScore: viterbi,
88 | origin: statePredecessor
89 | };
90 | changes.push(change);
91 | });
92 | });
93 | });
94 | newStates.forEach(ss => stateSets.getOrCreate(ss.position, ss.ruleStartPosition, ss.ruleDotPosition, ss.rule));
95 | }
96 | return changes;
97 | }
98 |
--------------------------------------------------------------------------------
/src/earley/scan.ts:
--------------------------------------------------------------------------------
1 | import { isNonTerminal, WordWithTypes, Terminal } from "../grammar/category";
2 | import { Semiring } from "semiring";
3 | import { Chart } from "./chart/chart";
4 | import { getActiveCategory, State, advanceDot } from "./chart/state";
5 |
6 |
7 | /**
8 | * Handles a token scanned from the input string.
9 | *
10 | * @param tokenPosition The start index of the scan.
11 | * @param word
12 | * @param types
13 | * @param scanProbability Function that provides the probability of scanning the given token at this position. Might be null for a probability of 1.0.
14 | * @param sr
15 | * @param stateSets
16 | */
17 | export function scan(tokenPosition: number,
18 | {word, types}: WordWithTypes,
19 | sr: Semiring,
20 | stateSets: Chart,
21 | scanProbability?: (x: T, t: Terminal[]) => S) {
22 | const changes: any[] = [];
23 |
24 | const scanProb: S = !!scanProbability ? scanProbability(word, types) : undefined;
25 |
26 | /*
27 | * Get all states that are active on a terminal
28 | * O(|stateset(i)|) = O(|grammar|): For all states i: Xk → λ·tμ, where t is a terminal that matches the given token...
29 | */
30 | types.forEach(terminal => {
31 | const statesActiveOnTerminals: Set> = stateSets.getStatesActiveOnTerminals(tokenPosition, terminal);
32 | if (statesActiveOnTerminals) statesActiveOnTerminals.forEach((preScanState: State) => {
33 | const activeCategory = getActiveCategory(preScanState);
34 | if (isNonTerminal(activeCategory)) throw new Error("this is a bug");
35 | else {
36 | if (!activeCategory(word)) throw new Error("Index failed");
37 | // TODO can this be more efficient, ie have tokens make their category be explicit? (Do we want to maintain the possibility of such "fluid" categories?)
38 | // Create the chart i+1: Xk → λt·μ
39 | const preScanForward: S = stateSets.getForwardScore(preScanState);
40 | const preScanInner: S = stateSets.getInnerScore(preScanState);
41 | // Note that this chart is unique for each preScanState
42 | const postScanState: State = stateSets.getOrCreate(
43 | tokenPosition + 1, preScanState.ruleStartPosition,
44 | advanceDot(preScanState),
45 | preScanState.rule,
46 | word
47 | );
48 |
49 | const postScanForward = calculateForwardScore(sr, preScanForward, scanProb);
50 | // Set forward score
51 | stateSets.setForwardScore(
52 | postScanState,
53 | postScanForward
54 | );
55 |
56 | // Get inner score (no side effects)
57 | const postScanInner: S = calculateInnerScore(sr, preScanInner, scanProb);
58 |
59 | // Set inner score
60 | stateSets.setInnerScore(
61 | postScanState,
62 | postScanInner
63 | );
64 |
65 | // Set Viterbi score
66 | const viterbiScore = {
67 | origin: preScanState,
68 | resultingState: postScanState,
69 | innerScore: postScanInner
70 | };
71 | stateSets.setViterbiScore(viterbiScore);
72 |
73 | changes.push({
74 | state: postScanState,
75 | viterbi: viterbiScore,
76 | inner: postScanInner,
77 | forward: postScanForward
78 | });
79 | }
80 | });
81 | });
82 | return changes;
83 | }
84 |
85 | /**
86 | * Function to calculate the new inner score from given values
87 | *
88 | * @param scanProbability The probability of scanning this particular token
89 | * @param sr The semiring to calculate with
90 | * @param previousInner The previous inner score
91 | * @return The inner score for the new chart
92 | */
93 | function calculateInnerScore(sr: Semiring, previousInner: S, scanProbability?: S): S {
94 | if (!scanProbability)
95 | return previousInner;
96 | else
97 | return sr.times(previousInner, scanProbability);
98 | }
99 |
100 | /**
101 | * Function to compute the forward score for the new chart after scanning the given token.
102 | *
103 | * @param scanProbability The probability of scanning this particular token
104 | * @param sr The semiring to calculate with
105 | * @param previousStateForwardScore The previous forward score
106 | * @return Computed forward score for the new chart
107 | */
108 | function calculateForwardScore(sr: Semiring, previousStateForwardScore: S, scanProbability?: S): S {
109 | if (!scanProbability) {
110 | return previousStateForwardScore;
111 | } else {
112 | return sr.times(previousStateForwardScore, scanProbability);
113 | }
114 | }
--------------------------------------------------------------------------------
/src/grammar/category.ts:
--------------------------------------------------------------------------------
1 | export type Category = Terminal | NonTerminal;
2 | export type Terminal = (t: T) => boolean;
3 | export type NonTerminal = string;
4 |
5 | export interface WordWithTypes {
6 | types: Terminal[];
7 | word: T;
8 | }
9 |
10 | export function isNonTerminal(element: any): element is NonTerminal {
11 | return typeof element === "string";
12 | }
--------------------------------------------------------------------------------
/src/grammar/grammar.ts:
--------------------------------------------------------------------------------
1 | import { NonTerminal, Category, isNonTerminal, Terminal } from "./category";
2 | import { Rule } from "./rule";
3 | import {
4 | getLeftCorners,
5 | getUnitStarCorners,
6 | getReflexiveTransitiveClosure,
7 | LeftCorners
8 | } from "./left-corner";
9 |
10 | import { Semiring, LogSemiring, makeDeferrable } from "semiring";
11 | import { Expression } from "semiring";
12 |
13 | function getOrCreateSet(map: Map